[2025-08-15 10:16:00.004202] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F5-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:00.005485] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17739][T1_TimerWK2_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:16:00.006690] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F5-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:00.006799] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526A-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:00.009319] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526A-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:00.089898] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A0-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:00.092311] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5309-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:00.093538] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A0-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:00.095201] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5309-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:00.105035] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58E9-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.137355] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0CA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.149741] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF06B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.169240] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FAF-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:00.171710] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FAF-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:00.186526] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D22-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.195492] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C4-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224160195348000, start_id=1755224160195348000, end_id=1755224160195348001) [2025-08-15 10:16:00.196169] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C4-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224160195348000, v:0}, tenant_id=1007) [2025-08-15 10:16:00.201062] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9D7-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.207801] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B3-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:00.210361] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B3-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:00.280201] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BE0-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87304688GB}) [2025-08-15 10:16:00.280249] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BE0-0-0] [lt=30] [DDL] execute ddl like stmt(ret=0, cost=53, ddl_arg=NULL) [2025-08-15 10:16:00.316589] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [17740][T1_TimerWK3_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=13] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:00.316622] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [17740][T1_TimerWK3_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=22] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:00.316639] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [17740][T1_TimerWK3_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=12] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:00.316662] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=17] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:00.379317] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=16][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227760378128}) [2025-08-15 10:16:00.379334] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:00.379344] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:00.385296] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227760384125}) [2025-08-15 10:16:00.385311] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:00.385320] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:00.386515] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227760385333}) [2025-08-15 10:16:00.386529] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:00.386537] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:00.400011] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7A-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.400033] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A22-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.402556] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7A-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:00.402573] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A22-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:00.402573] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.402591] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A22-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.442386] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6881-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.461408] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:00.461423] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:00.461428] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:00.502147] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E737-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.502150] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0C8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.509597] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF21F-0-0] [lt=10] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.515282] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F5-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.516626] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7863-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.517020] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.517390] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.517887] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F5-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.519211] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D4-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.519709] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7863-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.519887] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F5-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.519984] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F5-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.520200] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2A-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.520436] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05C-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.521666] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7863-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.521774] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7863-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.522204] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.522276] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D4-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.522299] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.522415] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.522497] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.524282] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D4-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.524367] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D4-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.529606] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A95-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83398438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83789062GB}) [2025-08-15 10:16:00.529651] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A95-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=50, ddl_arg=NULL) [2025-08-15 10:16:00.533473] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:00.533484] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:00.533489] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:00.534164] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:00.534181] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:00.534188] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:00.539984] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:00.539997] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:00.540002] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:00.545543] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:00.545554] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:00.545559] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:00.550955] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:00.550966] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:00.550971] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:00.587881] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7653-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.591056] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7653-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.592481] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885D-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.593001] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7653-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.593217] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7653-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.595092] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885D-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:00.595108] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885D-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.597257] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3C-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.599775] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3C-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:00.599789] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3C-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.623432] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:00.624656] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224160623356, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:16:00.624682] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224165624680, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:16:00.691190] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6884-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.702576] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD96-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.705547] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [3414065][T1006_TimerWK4_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=7] mview_mds_op(tx_count=0) [2025-08-15 10:16:00.705561] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [3414065][T1006_TimerWK4_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=9] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=851) [2025-08-15 10:16:00.707444] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [3414065][T1006_TimerWK4_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=6] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:00.707657] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:00.707669] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:00.707676] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:00.751575] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDA5-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.751641] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF06F-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.849304] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D25-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.870127] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B6-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.872743] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B6-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:00.872762] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B6-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.889014] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D4-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:00.890271] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8882-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.892745] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D4-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:00.892899] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8882-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:00.892916] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8882-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.922847] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB49-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.924009] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=12] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224160537181, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224159354201005, replayable_scn:1755224159354201005, readable_scn:1755224159354201005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:00.925380] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB49-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:00.925398] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB49-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:00.934739] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B37F-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:00.939154] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8384-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.942208] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8384-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.944307] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8384-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.944388] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8384-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:00.948002] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:00.949252] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224160947922, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:16:00.949278] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224165949276, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:16:00.992674] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58F0-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:00.993411] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9D3-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.012423] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F6-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:01.015127] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F6-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:01.016712] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526B-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:01.019433] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526B-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:01.026775] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:01.028669] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=22] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224161017899, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:16:01.028705] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=26] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224166028701, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:16:01.028730] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224166028727, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:01.100739] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A1-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:01.102919] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD530A-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:01.104447] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A1-0-0] [lt=1] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:01.105112] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58EA-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.105921] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD530A-0-0] [lt=0] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:01.137431] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0CB-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.149816] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF06C-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.177233] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB0-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:01.179860] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB0-0-0] [lt=0] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:01.186605] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D23-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.190897] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:01.192133] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224161190816, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:16:01.192158] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224166192156, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:16:01.193323] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:01.194505] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224161193249, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:16:01.194525] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224166194524, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:01.201132] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9D8-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.215786] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B4-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:01.218412] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B4-0-0] [lt=1] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:01.384225] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [727235][T1015_BackupDat][T1015][YB420ACB0122-000633A34EF59997-0-0] [lt=13] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:01.385740] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [727234][T1015_BACKUP_SC][T1015][YB420ACB0122-000633A34EE2C268-0-0] [lt=9] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:16:01.405298] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7B-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.405625] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A23-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.407924] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7B-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:01.407924] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E51-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224161407123000, start_id=1755224161407123000, end_id=1755224161407123001) [2025-08-15 10:16:01.407949] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7B-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.408479] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A23-0-0] [lt=93] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:01.408497] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A23-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.408755] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E51-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224161407123000, v:0}, tenant_id=1018) [2025-08-15 10:16:01.442472] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6882-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.454696] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438128][T1005_DRService][T1005][YB420ACB0122-0006338209C70E46-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1005, thread_idx=1) [2025-08-15 10:16:01.458459] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [438128][T1005_DRService][T1005][YB420ACB0122-0006338209C70E47-0-0] [lt=18] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2495, tenant_id=1005, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:01.460888] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [438128][T1005_DRService][T1005][YB420ACB0122-0006338209C70E47-0-0] [lt=14] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2411, tenant_id=1006, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:01.462688] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:01.462705] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:01.462716] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:01.469347] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D0A-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}) [2025-08-15 10:16:01.469385] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D0A-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=40, ddl_arg=NULL) [2025-08-15 10:16:01.502223] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E738-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.502223] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0C9-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.520931] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F6-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.522725] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7864-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.523293] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.523443] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05D-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.524032] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F6-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.525253] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D5-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224161357600001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224161357600002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224161357600002, v:0}}], last_dump_ts_=1755224156494484, force_dump=false) [2025-08-15 10:16:01.525281] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D5-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.526067] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F6-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.526219] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F6-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.526402] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2B-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.526430] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05D-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224161357600000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224161357600000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224161357600000, v:0}}], last_dump_ts_=1755224156495429, force_dump=false) [2025-08-15 10:16:01.526455] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05D-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.526722] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7864-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.528206] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D5-0-0] [lt=15] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224161357600001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224161357600002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224161357600002, v:0}}], last_dump_ts_=1755224156497599, force_dump=false) [2025-08-15 10:16:01.528233] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D5-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.528419] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.528418] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05D-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.528498] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.528531] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.529388] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7864-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.529484] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7864-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.530234] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D5-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.530331] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D5-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.533460] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:01.533475] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:01.533483] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:01.534198] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:01.534210] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:01.534215] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:01.540104] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:01.540118] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:01.540123] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:01.545563] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:01.545576] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:01.545581] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:01.551815] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:01.551831] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:01.551842] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:01.594234] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7654-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.597475] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7654-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.597834] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.599673] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7654-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.599765] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7654-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.600477] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885E-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:01.600495] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.602423] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3D-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.604926] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3D-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:01.604960] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3D-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.612261] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1015, thread_idx=0) [2025-08-15 10:16:01.613549] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1015, only_for_display=false) [2025-08-15 10:16:01.614792] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:01.628906] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.630652] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.631964] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.631988] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1015, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:01.632005] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.632016] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1015, ls_id:{id:1}, replicas:[{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:01.632119] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=80] build ls locality stat map(tenant_id=1015, ls_id={id:1}) [2025-08-15 10:16:01.632127] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.632139] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.632150] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.632161] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:01.632239] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.632254] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.632266] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.632279] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=9] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.632290] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=4563, tenant_id=1015, ls_id={id:1}) [2025-08-15 10:16:01.632303] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=10] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:01.633393] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19845, tenant_id=1015, acc_dr_task=0) [2025-08-15 10:16:01.634632] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1016, only_for_display=false) [2025-08-15 10:16:01.635840] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:01.649862] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.651020] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.652190] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.652212] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:01.652225] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=6] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.652236] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1}, replicas:[{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:01.652340] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=81] build ls locality stat map(tenant_id=1016, ls_id={id:1}) [2025-08-15 10:16:01.652347] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.652365] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=14] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.652378] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=10] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.652385] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:01.652468] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.652487] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.652498] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.652503] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.652512] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3811, tenant_id=1016, ls_id={id:1}) [2025-08-15 10:16:01.652519] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:01.653605] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:01.674751] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.675928] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.677077] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.677103] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:01.677116] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.677127] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1001}, replicas:[{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:01.677231] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=80] build ls locality stat map(tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:16:01.677239] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.677248] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=7] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.677254] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.677266] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:01.677343] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.677356] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.677366] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.677373] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.677383] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3792, tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:16:01.677397] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=10] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:01.678519] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:01.691268] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6885-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.699715] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.700869] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.702037] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.702059] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:01.702073] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.702084] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1002}, replicas:[{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:01.702188] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=81] build ls locality stat map(tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:16:01.702196] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.702210] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.702219] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.702230] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:01.702307] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.702321] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.702331] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.702336] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.702345] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3780, tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:16:01.702351] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:01.702646] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD97-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.703466] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:01.707753] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:01.707771] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:01.707778] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:01.724503] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.725663] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.727021] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:01.727053] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:01.727076] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=14] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.727091] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=12] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1003}, replicas:[{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:01.727272] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=143] build ls locality stat map(tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:16:01.727285] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.727304] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=14] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.727320] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=12] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:01.727337] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:01.727484] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=111] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.727506] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=12] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.727520] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=10] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.727534] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=11] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:01.727547] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=9] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=4218, tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:16:01.727562] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=11] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:01.729021] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=95611, tenant_id=1016, acc_dr_task=0) [2025-08-15 10:16:01.751646] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDA6-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.751718] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF070-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.768156] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC646-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.782288] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709535][T1014_TimerWK2_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=6] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:01.802237] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA1-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224161801795000, start_id=1755224161801795000, end_id=1755224161801795001) [2025-08-15 10:16:01.803073] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA1-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224161801795000, v:0}, tenant_id=1014) [2025-08-15 10:16:01.812548] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=15] mview_mds_op(tx_count=0) [2025-08-15 10:16:01.812564] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=12] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=830) [2025-08-15 10:16:01.813799] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475908][T1008_TimerWK0_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:01.832338] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:01.833580] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224161832260, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:16:01.833607] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224166833605, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:16:01.849373] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D26-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.875475] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B7-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.878058] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B7-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:01.878074] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B7-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.895632] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8883-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.898139] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D5-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:01.898227] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8883-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:01.898242] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8883-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.900690] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D5-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:01.903391] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [476032][T1007_BackupDat][T1007][YB420ACB0122-0006338212777C1D-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:01.920261] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [476031][T1007_BACKUP_SC][T1007][YB420ACB0122-000633821262C242-0-0] [lt=13] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:16:01.928195] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4A-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.930743] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4A-0-0] [lt=68] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:01.930762] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4A-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:01.936126] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B380-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:01.945429] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8385-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.948576] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8385-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.950775] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8385-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.950896] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8385-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:01.992751] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58F1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:01.993487] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9D4-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.020422] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F7-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:02.022926] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F7-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:02.024698] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526C-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:02.027178] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526C-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:02.105190] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58EB-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.111648] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A2-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:02.113557] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD530B-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:02.115427] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A2-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:02.116681] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD530B-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:02.137502] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0CC-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.148292] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727133][T1016_TimerWK1_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:02.149890] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF06D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.185099] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB1-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:02.186684] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D24-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.187591] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB1-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:02.201208] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9D9-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.208023] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727956][T1018_TimerWK0_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:02.223703] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B5-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:02.226196] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B5-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:02.262332] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619488][T1012_TimerWK3_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=28] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:16:02.280732] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B6360-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87304688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}) [2025-08-15 10:16:02.280778] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B6360-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=50, ddl_arg=NULL) [2025-08-15 10:16:02.322642] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487605][T1004_TimerWK2_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:02.337210] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:02.338455] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224162337136, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:16:02.338480] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224167338478, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:16:02.410692] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7C-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.411238] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A24-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.413220] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7C-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:02.413237] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7C-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.413826] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A24-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:02.413841] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A24-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.423517] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:02.424717] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224162423443, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:16:02.424743] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224167424741, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:16:02.442546] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6883-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.461408] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224160856808001, replayable_scn:1755224160856808001, readable_scn:1755224160856808001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224162458695, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224160856808001, replayable_scn:1755224160856808001, readable_scn:1755224160856808001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224162458695, ora_rowscn_=1755224161946803000, content_changed=true) [2025-08-15 10:16:02.461621] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:02.461633] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:02.461639] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:02.502300] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0CA-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.502302] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E739-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.509867] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF220-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.527203] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F7-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.529535] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.529533] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05E-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224162358831001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224162358831001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224162358831001, v:0}}], last_dump_ts_=1755224157498487, force_dump=false) [2025-08-15 10:16:02.529560] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05E-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.529730] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F7-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.530114] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1298-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83789062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84179688GB}) [2025-08-15 10:16:02.530153] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1298-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:16:02.530478] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7865-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.531277] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D6-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.531711] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F7-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.531811] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F7-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.533435] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224160445726001, replayable_scn:1755224160445726001, readable_scn:1755224160445726001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224162530272, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224160445726001, replayable_scn:1755224160445726001, readable_scn:1755224160445726001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224162530272, ora_rowscn_=1755224161595659000, content_changed=true) [2025-08-15 10:16:02.533555] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05E-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.533554] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2C-0-0] [lt=24] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.533620] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:02.533633] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:02.533638] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:02.534069] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224160356387000, replayable_scn:1755224160356387000, readable_scn:1755224160356387000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224162531328, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224160356387000, replayable_scn:1755224160356387000, readable_scn:1755224160356387000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224162531328, ora_rowscn_=1755224161527164000, content_changed=false) [2025-08-15 10:16:02.534135] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7865-0-0] [lt=15] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224162358831002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224162358831002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224162358831002, v:0}}], last_dump_ts_=1755224157501221, force_dump=false) [2025-08-15 10:16:02.534166] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7865-0-0] [lt=23] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.534684] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D6-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.535410] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:02.536212] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.536233] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2C-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.536302] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05E-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.536414] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.536582] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7865-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.536671] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7865-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.537071] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D6-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.537148] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D6-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.540002] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224161357600004, replayable_scn:1755224161357600004, readable_scn:1755224161357600004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224162536601, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224161357600004, replayable_scn:1755224161357600004, readable_scn:1755224161357600004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224162536601, ora_rowscn_=1755224162528164000, content_changed=true) [2025-08-15 10:16:02.540179] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:02.540192] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:02.540198] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:02.545489] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224161357600001, replayable_scn:1755224161357600001, readable_scn:1755224161357600001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224162542926, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224161357600001, replayable_scn:1755224161357600001, readable_scn:1755224161357600001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224162542926, ora_rowscn_=1755224162532382000, content_changed=true) [2025-08-15 10:16:02.545656] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:02.545671] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:02.545678] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:02.547021] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:02.550913] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224161357600000, replayable_scn:1755224161357600000, readable_scn:1755224161357600000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224162548192, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224161357600000, replayable_scn:1755224161357600000, readable_scn:1755224161357600000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224162548192, ora_rowscn_=1755224162532382000, content_changed=true) [2025-08-15 10:16:02.551089] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:02.551100] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:02.551105] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:02.600730] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7655-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.603198] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885F-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.603749] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7655-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.605743] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885F-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:02.605749] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7655-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.605763] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885F-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.605947] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7655-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.607663] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3E-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.610203] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3E-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:02.610220] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.691343] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6886-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.702125] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1013, thread_idx=0) [2025-08-15 10:16:02.702724] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD98-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.703392] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1013, only_for_display=false) [2025-08-15 10:16:02.704615] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:02.707675] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224161357600001, replayable_scn:1755224161357600001, readable_scn:1755224161357600001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224162704048, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224161357600001, replayable_scn:1755224161357600001, readable_scn:1755224161357600001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224162704048, ora_rowscn_=1755224162532382000, content_changed=true) [2025-08-15 10:16:02.707861] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:02.707872] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:02.707879] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:02.718717] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.719883] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.721055] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.721078] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1013, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:02.721095] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.721106] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1013, ls_id:{id:1}, replicas:[{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:02.721210] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=81] build ls locality stat map(tenant_id=1013, ls_id={id:1}) [2025-08-15 10:16:02.721218] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.721231] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.721242] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.721254] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:02.721330] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.721345] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.721355] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.721365] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.721371] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3852, tenant_id=1013, ls_id={id:1}) [2025-08-15 10:16:02.721382] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:02.722527] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19135, tenant_id=1013, acc_dr_task=0) [2025-08-15 10:16:02.723751] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1014, only_for_display=false) [2025-08-15 10:16:02.724973] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:02.738807] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.739974] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.741126] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.741148] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:02.741165] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.741178] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1}, replicas:[{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:02.741289] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=87] build ls locality stat map(tenant_id=1014, ls_id={id:1}) [2025-08-15 10:16:02.741297] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.741310] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.741321] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.741332] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:02.741409] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.741424] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.741434] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.741444] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.741450] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3825, tenant_id=1014, ls_id={id:1}) [2025-08-15 10:16:02.741460] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:02.742553] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:02.751705] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDA7-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.751793] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF071-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.763760] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.764924] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.766089] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.766113] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:02.766131] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.766142] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1001}, replicas:[{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:02.766250] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=84] build ls locality stat map(tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:16:02.766258] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.766271] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.766281] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.766293] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:02.766371] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.766385] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.766394] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.766401] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.766409] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3797, tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:16:02.766419] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:02.767482] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:02.775636] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=13] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:02.775670] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=22] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:02.775684] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:02.777156] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [727134][T1016_TimerWK2_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=16] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:02.788583] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.789752] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.790913] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.790936] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:02.790961] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=16] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.790968] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=5] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1002}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:02.791073] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=81] build ls locality stat map(tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:16:02.791081] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.791095] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.791106] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.791117] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:02.791202] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.791216] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.791226] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.791232] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.791241] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3811, tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:16:02.791250] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:02.792334] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:02.802009] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:02.803249] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224162801934, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:16:02.803277] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224167803275, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:16:02.803359] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:02.804520] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224162803166, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:16:02.804539] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224167804538, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:16:02.813321] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.814455] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.815617] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:02.815640] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:02.815654] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.815664] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1003}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:02.815772] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=85] build ls locality stat map(tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:16:02.815785] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.815797] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.815805] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:02.815813] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:02.815889] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.815901] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.815909] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.815916] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:02.815923] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3748, tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:16:02.815931] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:02.817071] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=9] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94526, tenant_id=1014, acc_dr_task=0) [2025-08-15 10:16:02.849443] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D27-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.880831] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B8-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.883415] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B8-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:02.883433] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B8-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.900989] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8884-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.903611] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8884-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:02.903634] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8884-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.906055] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D6-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:02.908591] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D6-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:02.933563] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4B-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.936074] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4B-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:02.936092] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:02.937536] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B381-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:02.951877] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8386-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.954510] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:340) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8386-0-0] [lt=11] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1014, ls_id:{id:1}, sync_scn:{val:1755224162859762000, v:0}, readable_scn:{val:1755224162859762000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-08-15 10:16:02.954863] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8386-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.956908] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8386-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.957037] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8386-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:02.957076] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:459) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8386-0-0] [lt=7] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1755224160856808001, v:0}) [2025-08-15 10:16:02.965362] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:02.966604] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224162965281, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:16:02.966626] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224167966624, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:16:02.992824] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58F2-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:02.993540] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9D5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.028321] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F8-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:03.030837] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F8-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:03.032461] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526D-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:03.033653] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526D-0-0] [lt=13] log restore source is empty, just skip(ret=-4018, user_tenant_id=1018) [2025-08-15 10:16:03.036115] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526D-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:03.051088] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:03.052316] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224163051040, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:16:03.052341] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224168052339, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:16:03.105264] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58EC-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.122650] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A3-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:03.124429] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD530C-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:03.126319] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A3-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:03.127558] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD530C-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:03.137572] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0CD-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.149965] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF06E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.186756] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D25-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.192891] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB2-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:03.196563] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB2-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:03.201281] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9DA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.231574] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B6-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:03.234064] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B6-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:03.390883] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227763389679}) [2025-08-15 10:16:03.390902] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:03.390911] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:03.396857] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=17][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227763395679}) [2025-08-15 10:16:03.396873] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:03.396880] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:03.398070] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227763396892}) [2025-08-15 10:16:03.398085] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:03.398093] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=5][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:03.415956] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7D-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.416510] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A25-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.418520] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7D-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:03.418538] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.419102] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A25-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:03.419121] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A25-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.442621] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6884-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.461678] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:03.461695] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:03.461705] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:03.461717] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1014, cost_time_us=204, return_code_array=[0, 0, 0], broadcast_times=2) [2025-08-15 10:16:03.469830] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D0C-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}) [2025-08-15 10:16:03.469867] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D0C-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=40, ddl_arg=NULL) [2025-08-15 10:16:03.502374] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0CB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.502376] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E73A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.532782] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F8-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224163360500002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224163360500002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224163360500002, v:0}}], last_dump_ts_=1755224158503717, force_dump=false) [2025-08-15 10:16:03.532808] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F8-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.533652] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:03.533665] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:03.533678] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:03.534880] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:03.534892] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:03.534898] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:03.535192] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F8-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224163360500002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224163360500002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224163360500002, v:0}}], last_dump_ts_=1755224158506465, force_dump=false) [2025-08-15 10:16:03.535215] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F8-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.537125] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F8-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.537242] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F8-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.537272] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.537331] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.537641] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7866-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.538063] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D7-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.540205] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:03.540215] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:03.540221] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:03.540684] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7866-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.541075] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D7-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.541189] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2D-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.541208] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05F-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.542770] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7866-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.542887] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7866-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.543180] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.543243] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D7-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.543279] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.543347] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D7-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.543377] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05F-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.543470] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.545717] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:03.545728] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:03.545734] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:03.551121] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:03.551131] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:03.551136] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:03.606946] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7656-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.608509] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8860-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.609141] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:03.609956] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7656-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.610379] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224163609064, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:16:03.610405] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224168610403, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:16:03.611101] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8860-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:03.611119] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8860-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.611868] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7656-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.611957] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7656-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.612908] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3F-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.615402] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3F-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:03.615417] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3F-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.691418] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6887-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.701046] INFO [RS] finish (ob_mview_maintenance_task.cpp:280) [709536][T1014_TimerWK3_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=13] mview maintenace task finish(tenant_id_=1014, round_=63450, status_=100, error_code_=0, cost_us_=20001383, prepare_cost_us_=0, gc_mview_cost_us_=1220, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:16:03.702803] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD99-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.707839] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:03.707852] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:03.707858] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:03.751778] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDA8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.751870] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF072-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.773090] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC647-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.849511] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D28-0-0] [lt=8] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.886202] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B9-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.888816] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B9-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:03.888835] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B9-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.906362] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8885-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.908916] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8885-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:03.908935] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8885-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.913937] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D7-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:03.916458] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D7-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:03.925574] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224163537251, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224162358831002, replayable_scn:1755224162358831002, readable_scn:1755224162358831002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:03.938874] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4C-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.938963] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B382-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:03.941387] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4C-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:03.941404] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4C-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:03.958077] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8387-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.961009] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8387-0-0] [lt=9] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224163861077001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224163861077001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224163861077001, v:0}}], last_dump_ts_=1755224158929370, force_dump=false) [2025-08-15 10:16:03.961030] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8387-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.962937] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8387-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.963116] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8387-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:03.992895] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58F3-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:03.993619] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9D6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.036174] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F9-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:04.038663] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F9-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:04.041480] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526E-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:04.044033] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526E-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:04.105342] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58ED-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.131677] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A4-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:04.132831] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD530D-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:04.134870] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A4-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:04.136501] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD530D-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:04.137642] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0CE-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.140314] INFO [RS] prepare (ob_mview_maintenance_task.cpp:176) [438023][T1006_TimerWK2_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=15] mview maintenance task prepare success(tenant_id_=1006, round_=105668, prepare_cost_us_=0) [2025-08-15 10:16:04.150039] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF06F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.186826] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D26-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.201356] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9DB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.201876] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB3-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:04.204385] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB3-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:04.239433] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B7-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:04.243078] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B7-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:04.281259] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BE4-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}) [2025-08-15 10:16:04.281304] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BE4-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:16:04.421281] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7E-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.421762] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A26-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.423853] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7E-0-0] [lt=68] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:04.423868] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7E-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.424271] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A26-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:04.424285] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A26-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.442656] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6885-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.461706] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:04.461717] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:04.461723] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:04.463032] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:04.502448] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0CC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.502448] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E73B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.510151] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF221-0-0] [lt=10] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.533684] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:04.533698] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:04.533709] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:04.534245] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:04.534259] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:04.534265] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:04.535115] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:04.537598] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D129D-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84179688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84570312GB}) [2025-08-15 10:16:04.537639] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D129D-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:16:04.538285] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F9-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.541284] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F9-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.543339] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.543520] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.543782] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7867-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224164361748001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224164361748001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224164361748001, v:0}}], last_dump_ts_=1755224159510313, force_dump=false) [2025-08-15 10:16:04.543803] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7867-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.544201] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D8-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.544205] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2E-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224164265003000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224164265003000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224164265003000, v:0}}], last_dump_ts_=1755224159510534, force_dump=false) [2025-08-15 10:16:04.544227] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2E-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.544402] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC060-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.546780] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7867-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.547099] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D8-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.547140] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2E-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224164265003000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224164265003000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224164265003000, v:0}}], last_dump_ts_=1755224159513725, force_dump=false) [2025-08-15 10:16:04.547165] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2E-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.547398] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC060-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.548840] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7867-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.548934] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7867-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.549052] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D8-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.549094] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2E-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.549151] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D8-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.549177] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.549405] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC060-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.549492] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC060-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.551205] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:04.551217] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:04.551222] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:04.552542] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:04.590718] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:104) [17458][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=11] start checking lost replicas [2025-08-15 10:16:04.612902] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7657-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224164460683001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224164460683001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224164460683001, v:0}}], last_dump_ts_=1755224159581341, force_dump=false) [2025-08-15 10:16:04.612927] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7657-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.613826] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8861-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.615891] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7657-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224164460683000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224164460683000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224164460683000, v:0}}], last_dump_ts_=1755224159584551, force_dump=false) [2025-08-15 10:16:04.615913] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7657-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.616357] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8861-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:04.616376] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8861-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.617827] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7657-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.617917] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7657-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.618133] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A40-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.620617] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A40-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:04.620634] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A40-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.629760] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:04.630990] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224164629681, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:16:04.631015] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224169631013, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:16:04.691495] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6888-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.702877] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD9A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.707851] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:04.707864] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:04.707870] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:04.709147] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:04.717122] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB92-0-0] [lt=15] [RESTORE] restore scheduler quit [2025-08-15 10:16:04.718389] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB92-0-0] [lt=10] get targets(ret=0, jobs=[], exec_tenant_id=1013, sql=select * from __all_recover_table_job) [2025-08-15 10:16:04.719648] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB93-0-0] [lt=15] get targets(ret=0, jobs=[], exec_tenant_id=1013, sql=select * from __all_import_table_job) [2025-08-15 10:16:04.719673] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB94-0-0] [lt=16][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224164719669, ref_timestamp=1755224104715209, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:16:04.719763] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB95-0-0] [lt=13] [RESTORE] restore scheduler start [2025-08-15 10:16:04.719776] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB96-0-0] [lt=10] [RESTORE] try process restore job [2025-08-15 10:16:04.720926] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB96-0-0] [lt=4] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:16:04.751852] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDA9-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.751947] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF073-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.840715] INFO [RS.LB] idle (ob_root_balancer.cpp:87) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=29] root balance idle(idle_time=10000000) [2025-08-15 10:16:04.840729] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:70) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=11] start check unit migrate finish [2025-08-15 10:16:04.840741] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=4] try check migrate unit finish by tenant(tenant_id=1) [2025-08-15 10:16:04.849587] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D29-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.850104] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=3] try check migrate unit finish by tenant(tenant_id=1004) [2025-08-15 10:16:04.875871] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1006) [2025-08-15 10:16:04.891620] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BA-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.894339] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BA-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:04.894357] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BA-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.901815] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1008) [2025-08-15 10:16:04.911665] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8886-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.914218] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8886-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:04.914235] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8886-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.921856] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D8-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:04.924319] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D8-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:04.927565] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1012) [2025-08-15 10:16:04.940387] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B383-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:04.944188] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4D-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.946734] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4D-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:04.946752] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:04.953252] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1014) [2025-08-15 10:16:04.964214] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8388-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224164862585000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224164862585000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224164862585000, v:0}}], last_dump_ts_=1755224159932616, force_dump=false) [2025-08-15 10:16:04.964237] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8388-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.967360] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8388-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.969284] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8388-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.969385] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8388-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:04.979037] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1016) [2025-08-15 10:16:04.992969] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58F4-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:04.993693] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9D7-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.004723] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1018) [2025-08-15 10:16:05.006822] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:05.028113] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [2619487][T1012_TimerWK2_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=10] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:05.028148] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [2619487][T1012_TimerWK2_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=22] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:05.028162] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [2619487][T1012_TimerWK2_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:05.029384] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [2619486][T1012_TimerWK1_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=9] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:05.030525] INFO [RS] check (ob_alter_locality_finish_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=14] start to check alter locality finish [2025-08-15 10:16:05.031704] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81538-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.032859] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81538-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.035165] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8153A-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.036313] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8153A-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.038589] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8153C-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.039740] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8153C-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.042050] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8153E-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.043226] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8153E-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.044037] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8FA-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:05.045541] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81540-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.046550] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8FA-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:05.046689] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81540-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.048954] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81542-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.049350] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526F-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:05.050122] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81542-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.051887] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD526F-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:05.052428] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81544-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.053580] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81544-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.055856] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81546-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.057004] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81546-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.058187] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81546-0-0] [lt=12] start check shrink resource pool [2025-08-15 10:16:05.058206] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81547-0-0] [lt=12] start check shrink resource pool(tenant_id=1) [2025-08-15 10:16:05.059323] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81547-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.060468] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81547-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.060522] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81548-0-0] [lt=14] start check shrink resource pool(tenant_id=1004) [2025-08-15 10:16:05.061642] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81548-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.062784] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81548-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.062851] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81549-0-0] [lt=13] start check shrink resource pool(tenant_id=1006) [2025-08-15 10:16:05.063972] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81549-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.065113] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81549-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.065165] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154A-0-0] [lt=12] start check shrink resource pool(tenant_id=1008) [2025-08-15 10:16:05.066273] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154A-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.067404] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154A-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.067456] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154B-0-0] [lt=13] start check shrink resource pool(tenant_id=1012) [2025-08-15 10:16:05.068569] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154B-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.069712] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154B-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.069774] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154C-0-0] [lt=12] start check shrink resource pool(tenant_id=1014) [2025-08-15 10:16:05.070894] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154C-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.072062] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154C-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.072116] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154D-0-0] [lt=12] start check shrink resource pool(tenant_id=1016) [2025-08-15 10:16:05.073226] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154D-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.074376] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154D-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.074428] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154E-0-0] [lt=12] start check shrink resource pool(tenant_id=1018) [2025-08-15 10:16:05.075552] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154E-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.076693] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154E-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.076745] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:99) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154E-0-0] [lt=12] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.076758] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154E-0-0] [lt=10] start check primary zone [2025-08-15 10:16:05.076767] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154F-0-0] [lt=5] start check primary zone(tenant_id=1) [2025-08-15 10:16:05.077876] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8154F-0-0] [lt=6][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.077926] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81550-0-0] [lt=11] start check primary zone(tenant_id=1004) [2025-08-15 10:16:05.079031] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81550-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.079083] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81551-0-0] [lt=13] start check primary zone(tenant_id=1006) [2025-08-15 10:16:05.080201] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81551-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.080252] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81552-0-0] [lt=12] start check primary zone(tenant_id=1008) [2025-08-15 10:16:05.081378] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81552-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.081434] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81553-0-0] [lt=13] start check primary zone(tenant_id=1012) [2025-08-15 10:16:05.082561] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81553-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.082615] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81554-0-0] [lt=12] start check primary zone(tenant_id=1014) [2025-08-15 10:16:05.083749] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81554-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.083804] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81555-0-0] [lt=12] start check primary zone(tenant_id=1016) [2025-08-15 10:16:05.084936] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81555-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.084994] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=15] start check primary zone(tenant_id=1018) [2025-08-15 10:16:05.086105] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:05.086159] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:166) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=12] start do unit balance [2025-08-15 10:16:05.092385] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:350) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=9] start distribute for server status change [2025-08-15 10:16:05.092493] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:401) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=13] finish distribute for server status change(ret=0) [2025-08-15 10:16:05.092508] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:200) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=12] finish do unit balance(ret=0) [2025-08-15 10:16:05.098415] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:115) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=9] start to do tenant group unit balance [2025-08-15 10:16:05.098456] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=12] start to do tenant group unit balance(zone=zone1) [2025-08-15 10:16:05.098471] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=9] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.203.1.34:2882", disk_in_use_:3105882112, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:05.098490] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=12] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:05.098519] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:05.098536] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=8] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:05.098547] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone1) [2025-08-15 10:16:05.098559] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=8] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:05.098580] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=13] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.1.34:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:05.098624] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=34] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.203.1.34:2882", disk_in_use_:3105882112, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:05.098636] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=9] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.1.34:2882", disk_in_use_:3105882112, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:05.098646] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=6] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:05.098659] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=10] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1006, unit_config:{unit_config_id:1005, name:"config_opsone_zone1_s_2_4_ywx", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7988, unit_config:0x7f1e9aa71c38, pool:0x7f1c3beb2f18}, {unit_id:1008, tenant_id:1006, resource_pool_id:1008, unit_config:{unit_config_id:1008, name:"config_test_zone1_s_2_4_emp", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f77d8, unit_config:0x7f1e9aa71998, pool:0x7f1c3beb27a8}, {unit_id:1011, tenant_id:1008, resource_pool_id:1011, unit_config:{unit_config_id:1011, name:"config_mcp_zone1_s_4_8_tmc", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7550, unit_config:0x7f1e9aa716f8, pool:0x7f1e443fda40}, {unit_id:1016, tenant_id:1012, resource_pool_id:1016, unit_config:{unit_config_id:1017, name:"config_oms_zone1_s_4_8_gzp", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f73a0, unit_config:0x7f1e9aa71458, pool:0x7f1e443fd2d0}, {unit_id:1019, tenant_id:1014, resource_pool_id:1019, unit_config:{unit_config_id:1019, name:"config_grafana_test_zone1_s_1_2_pri", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7118, unit_config:0x7f1e9aa71298, pool:0x7f1e443fc7a8}, {unit_id:1024, tenant_id:1016, resource_pool_id:1024, unit_config:{unit_config_id:1023, name:"config_temporalio_zone1_s_2_4_ykw", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6ce0, unit_config:0x7f1e9aa70f18, pool:0x7f1e74dff2d0}, {unit_id:1026, tenant_id:1018, resource_pool_id:1026, unit_config:{unit_config_id:1027, name:"config_media_zone1_s_2_4_kmn", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6b30, unit_config:0x7f1e9aa70b98, pool:0x7f1e74dfeb60}], not_grant_units=[], available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:05.098716] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=34] start to do tenant group unit balance(zone=zone2) [2025-08-15 10:16:05.098725] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.203.4.38:2882", disk_in_use_:3093299200, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:05.098737] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:05.098754] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=9] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:05.098766] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=7] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:05.098775] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone2) [2025-08-15 10:16:05.098780] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:05.098793] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=8] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.38:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:05.098825] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=25] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.203.4.38:2882", disk_in_use_:3093299200, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:05.098831] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=5] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.38:2882", disk_in_use_:3093299200, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:05.098842] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=7] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:05.098852] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=8] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1004, unit_config:{unit_config_id:1004, name:"config_opsone_zone2_s_2_4_wag", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7b38, unit_config:0x7f1e9aa71d18, pool:0x7f1c3beb3688}, {unit_id:1009, tenant_id:1006, resource_pool_id:1009, unit_config:{unit_config_id:1009, name:"config_test_zone2_s_2_4_yey", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7700, unit_config:0x7f1e9aa718b8, pool:0x7f1c3beb23f0}, {unit_id:1010, tenant_id:1008, resource_pool_id:1010, unit_config:{unit_config_id:1010, name:"config_mcp_zone2_s_4_8_bjb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7628, unit_config:0x7f1e9aa717d8, pool:0x7f1c3beb2038}, {unit_id:1018, tenant_id:1012, resource_pool_id:1018, unit_config:{unit_config_id:1016, name:"config_oms_zone2_s_4_8_kbx", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f71f0, unit_config:0x7f1e9aa71538, pool:0x7f1e443fcb60}, {unit_id:1021, tenant_id:1014, resource_pool_id:1021, unit_config:{unit_config_id:1021, name:"config_grafana_test_zone2_s_1_2_yyc", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f6f68, unit_config:0x7f1e9aa710d8, pool:0x7f1e443fc038}, {unit_id:1023, tenant_id:1016, resource_pool_id:1023, unit_config:{unit_config_id:1022, name:"config_temporalio_zone2_s_2_4_lsv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6db8, unit_config:0x7f1e9aa70ff8, pool:0x7f1e74dff688}, {unit_id:1027, tenant_id:1018, resource_pool_id:1027, unit_config:{unit_config_id:1025, name:"config_media_zone2_s_2_4_yzd", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6a58, unit_config:0x7f1e9aa70d58, pool:0x7f1e74dfe7a8}], not_grant_units=[], available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:05.098904] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=30] start to do tenant group unit balance(zone=zone3) [2025-08-15 10:16:05.098914] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.203.4.39:2882", disk_in_use_:3055550464, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:05.098925] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:05.098951] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=9] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:05.098963] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=7] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:05.098972] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone3) [2025-08-15 10:16:05.098976] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:05.098989] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84570312GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.39:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84570312GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:05.099020] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=24] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.203.4.39:2882", disk_in_use_:3055550464, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:05.099026] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=5] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.39:2882", disk_in_use_:3055550464, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:05.099037] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:05.099047] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=9] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1005, unit_config:{unit_config_id:1006, name:"config_opsone_zone3_s_2_4_vxv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7a60, unit_config:0x7f1e9aa71b58, pool:0x7f1c3beb32d0}, {unit_id:1007, tenant_id:1006, resource_pool_id:1007, unit_config:{unit_config_id:1007, name:"config_test_zone3_s_2_4_mui", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f78b0, unit_config:0x7f1e9aa71a78, pool:0x7f1c3beb2b60}, {unit_id:1012, tenant_id:1008, resource_pool_id:1012, unit_config:{unit_config_id:1012, name:"config_mcp_zone3_s_4_8_seu", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7478, unit_config:0x7f1e9aa71618, pool:0x7f1e443fd688}, {unit_id:1017, tenant_id:1012, resource_pool_id:1017, unit_config:{unit_config_id:1018, name:"config_oms_zone3_s_4_8_nvb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f72c8, unit_config:0x7f1e9aa71378, pool:0x7f1e443fcf18}, {unit_id:1020, tenant_id:1014, resource_pool_id:1020, unit_config:{unit_config_id:1020, name:"config_grafana_test_zone3_s_1_2_xbz", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7040, unit_config:0x7f1e9aa711b8, pool:0x7f1e443fc3f0}, {unit_id:1022, tenant_id:1016, resource_pool_id:1022, unit_config:{unit_config_id:1024, name:"config_temporalio_zone3_s_2_4_rxz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6e90, unit_config:0x7f1e9aa70e38, pool:0x7f1e74dffa40}, {unit_id:1025, tenant_id:1018, resource_pool_id:1025, unit_config:{unit_config_id:1026, name:"config_media_zone3_s_2_4_ukz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6c08, unit_config:0x7f1e9aa70c78, pool:0x7f1e74dfef18}], not_grant_units=[], available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:05.105420] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58EE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.124721] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1011, thread_idx=0) [2025-08-15 10:16:05.126012] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1011, only_for_display=false) [2025-08-15 10:16:05.127275] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:05.137717] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0CF-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.140212] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A5-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:05.141817] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD530E-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:05.142597] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.142710] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A5-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:05.143756] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.144316] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD530E-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:05.144929] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.144958] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1011, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:05.144975] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.144985] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1011, ls_id:{id:1}, replicas:[{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:05.145092] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=83] build ls locality stat map(tenant_id=1011, ls_id={id:1}) [2025-08-15 10:16:05.145103] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.145116] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.145125] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.145134] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:05.145213] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.145225] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.145233] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.145241] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.145249] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3820, tenant_id=1011, ls_id={id:1}) [2025-08-15 10:16:05.145257] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:05.146686] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=20674, tenant_id=1011, acc_dr_task=0) [2025-08-15 10:16:05.147911] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1012, only_for_display=false) [2025-08-15 10:16:05.149134] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:05.150087] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF070-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.164304] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.165451] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.166618] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.166639] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:05.166655] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.166665] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1}, replicas:[{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:05.166776] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=86] build ls locality stat map(tenant_id=1012, ls_id={id:1}) [2025-08-15 10:16:05.166785] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.166798] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.166806] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.166816] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:05.166895] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.166908] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.166917] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.166925] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.166933] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3858, tenant_id=1012, ls_id={id:1}) [2025-08-15 10:16:05.166948] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=11] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:05.168703] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:05.186898] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D27-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.191123] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.192287] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.193444] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.193466] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:05.193484] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.193497] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1001}, replicas:[{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:05.193607] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=85] build ls locality stat map(tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:16:05.193616] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.193629] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.193636] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.193647] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:05.193726] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.193740] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.193751] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.193757] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.193765] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3794, tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:16:05.193772] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:05.195203] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:05.201427] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9DC-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.209734] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB4-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:05.212276] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB4-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:05.217320] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.218465] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.219615] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.219638] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:05.219654] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.219663] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1002}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:05.219771] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=83] build ls locality stat map(tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:16:05.219781] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.219794] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.219803] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.219813] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:05.219891] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.219903] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.219912] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.219920] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.219928] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3749, tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:16:05.219937] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:05.221476] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:05.243676] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.244857] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.246007] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.246029] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:05.246047] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.246058] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1003}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:05.246166] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=83] build ls locality stat map(tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:16:05.246174] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.246186] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.246196] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.246206] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:05.246284] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.246298] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.246306] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.246314] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.246322] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3800, tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:16:05.246331] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:05.247930] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=101229, tenant_id=1012, acc_dr_task=0) [2025-08-15 10:16:05.248394] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B8-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:05.250914] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B8-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:05.286325] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E56-0-0] [lt=6] start do ObArchiveSchedulerService round(round=101515) [2025-08-15 10:16:05.289850] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E56-0-0] [lt=12][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:16:05.289865] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E56-0-0] [lt=12][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1014 AND dest_no = 0 AND name = 'state', exec_tenant_id=1013) [2025-08-15 10:16:05.289908] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E56-0-0] [lt=8][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1014, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:16:05.289921] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E56-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1014, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:16:05.289930] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E56-0-0] [lt=8][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1014, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:16:05.426681] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7F-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.426977] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A27-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.429212] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7F-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:05.429231] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF7F-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.429505] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A27-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:05.429523] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A27-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.442728] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6886-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.444497] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1007, thread_idx=0) [2025-08-15 10:16:05.445806] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=20] start try tenant disaster recovery(tenant_id=1007, only_for_display=false) [2025-08-15 10:16:05.447051] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:05.461040] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.461585] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224163861077001, replayable_scn:1755224163861077001, readable_scn:1755224163861077001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224165458003, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224163861077001, replayable_scn:1755224163861077001, readable_scn:1755224163861077001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224165458003, ora_rowscn_=1755224164965625000, content_changed=true) [2025-08-15 10:16:05.461776] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:05.461789] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:05.461794] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:05.462202] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.463437] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.463458] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1007, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:05.463473] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.463483] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1007, ls_id:{id:1}, replicas:[{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:05.463585] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=79] build ls locality stat map(tenant_id=1007, ls_id={id:1}) [2025-08-15 10:16:05.463595] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.463606] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.463614] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.463624] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:05.463699] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.463711] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.463718] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.463725] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.463732] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3860, tenant_id=1007, ls_id={id:1}) [2025-08-15 10:16:05.463740] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:05.464864] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19058, tenant_id=1007, acc_dr_task=0) [2025-08-15 10:16:05.466193] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1008, only_for_display=false) [2025-08-15 10:16:05.467401] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:05.470335] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A0930-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}) [2025-08-15 10:16:05.470376] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A0930-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:16:05.481312] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.482483] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.483636] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.483657] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:05.483672] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.483681] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1}, replicas:[{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:05.483794] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=88] build ls locality stat map(tenant_id=1008, ls_id={id:1}) [2025-08-15 10:16:05.483801] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.483812] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.483822] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.483832] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:05.483908] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.483920] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.483927] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.483935] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.483946] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=9] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3806, tenant_id=1008, ls_id={id:1}) [2025-08-15 10:16:05.483955] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:05.485055] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:05.502525] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0CD-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.502524] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E73C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.506022] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.507200] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.508520] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.508546] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:05.508564] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.508575] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1001}, replicas:[{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:05.508681] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=83] build ls locality stat map(tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:16:05.508690] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.508704] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.508710] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.508721] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:05.508799] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.508814] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.508823] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.508830] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.508839] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3987, tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:16:05.508850] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:05.509926] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:05.530752] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.531910] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.533091] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.533111] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:05.533125] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.533135] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1002}, replicas:[{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:05.533254] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=93] build ls locality stat map(tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:16:05.533265] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.533276] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.533285] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.533295] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:05.533376] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.533388] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.533395] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.533403] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.533410] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3794, tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:16:05.533417] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:05.533466] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224163360500001, replayable_scn:1755224163360500001, readable_scn:1755224163360500001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224165530541, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224163360500001, replayable_scn:1755224163360500001, readable_scn:1755224163360500001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224165530541, ora_rowscn_=1755224164614318000, content_changed=true) [2025-08-15 10:16:05.533660] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:05.533678] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:05.533684] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:05.534211] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224163360500001, replayable_scn:1755224163360500001, readable_scn:1755224163360500001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224165531598, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224163360500001, replayable_scn:1755224163360500001, readable_scn:1755224163360500001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224165531598, ora_rowscn_=1755224164545968000, content_changed=true) [2025-08-15 10:16:05.534389] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:05.534401] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:05.534406] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:05.534560] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:05.539735] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:05.539745] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:05.539753] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:05.541027] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:05.544569] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FA-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.545771] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224163359445003, replayable_scn:1755224163359445003, readable_scn:1755224163359445003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224165542107, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224163359445003, replayable_scn:1755224163359445003, readable_scn:1755224163359445003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224165542107, ora_rowscn_=1755224164545968000, content_changed=true) [2025-08-15 10:16:05.545959] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:05.545971] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:05.545979] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:05.547663] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FA-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.549520] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FA-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.549610] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FA-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.549933] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7868-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.550093] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.550104] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2F-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.550448] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC061-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.551100] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224163360500000, replayable_scn:1755224163360500000, readable_scn:1755224163360500000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224165548429, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224163360500000, replayable_scn:1755224163360500000, readable_scn:1755224163360500000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224165548429, ora_rowscn_=1755224164545968000, content_changed=false) [2025-08-15 10:16:05.553017] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7868-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.553028] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2F-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.553150] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D9-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.553368] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC061-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.554971] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7868-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.554977] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.555138] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C2F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.555153] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7868-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.555274] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.555362] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D9-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.555377] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC061-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.555462] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC061-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.555828] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.556990] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.558159] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:05.558186] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:05.558201] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.558210] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1003}, replicas:[{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:05.558311] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=79] build ls locality stat map(tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:16:05.558321] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.558332] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.558341] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:05.558350] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:05.558431] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.558443] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.558451] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.558457] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:05.558465] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3797, tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:16:05.558474] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:05.559580] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94697, tenant_id=1008, acc_dr_task=0) [2025-08-15 10:16:05.618913] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7658-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.619118] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8862-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.621692] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8862-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:05.621710] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8862-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.621791] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7658-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.623320] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A41-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.623656] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7658-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.623768] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7658-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.624757] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:05.625866] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A41-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:05.625884] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A41-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.626004] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224165624680, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:16:05.626037] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=25] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224170626036, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:16:05.691572] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6889-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.702953] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD9B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.707772] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224164361748001, replayable_scn:1755224164361748001, readable_scn:1755224164361748001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224165705124, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224164361748001, replayable_scn:1755224164361748001, readable_scn:1755224164361748001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224165705124, ora_rowscn_=1755224165551593000, content_changed=true) [2025-08-15 10:16:05.707954] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:05.707967] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:05.707972] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:05.711445] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438022][T1006_TimerWK1_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=8] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:16:05.751925] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDAA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.752056] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF074-0-0] [lt=41] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.777959] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC648-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.849662] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D2A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.897209] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BB-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.899853] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BB-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:05.899869] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BB-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.916971] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8887-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.919590] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8887-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:05.919609] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8887-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.929744] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D9-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:05.932572] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D9-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:05.941791] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B384-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:05.949358] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:05.949562] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4E-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.950591] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224165949276, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:16:05.950615] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224170950613, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:16:05.952142] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4E-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:05.952159] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:05.970427] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8389-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.973622] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8389-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.976124] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8389-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.976231] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8389-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:05.981797] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC129106A-0-0] [lt=15] [RESTORE] restore scheduler quit [2025-08-15 10:16:05.983048] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC129106A-0-0] [lt=12] get targets(ret=0, jobs=[], exec_tenant_id=1011, sql=select * from __all_recover_table_job) [2025-08-15 10:16:05.984239] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC129106B-0-0] [lt=13] get targets(ret=0, jobs=[], exec_tenant_id=1011, sql=select * from __all_import_table_job) [2025-08-15 10:16:05.984262] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC129106C-0-0] [lt=13][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224165984256, ref_timestamp=1755224105979630, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:16:05.984319] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC129106D-0-0] [lt=10] [RESTORE] restore scheduler start [2025-08-15 10:16:05.984332] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC129106E-0-0] [lt=10] [RESTORE] try process restore job [2025-08-15 10:16:05.985493] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC129106E-0-0] [lt=6] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:16:05.993044] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58F5-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:05.993769] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9D8-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.037548] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:06.038779] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224166028701, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:16:06.038802] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224171038801, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:16:06.038818] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224171038816, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:06.052177] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8FB-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:06.055849] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8FB-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:06.057190] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5270-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:06.059706] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5270-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:06.105500] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58EF-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.137792] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0D0-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.148117] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A6-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:06.150164] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF071-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.150338] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD530F-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:06.150643] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:06.152863] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD530F-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:06.186972] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D28-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.192234] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=6] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:06.193487] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224166192156, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:16:06.193512] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224171193510, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:16:06.194598] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:06.195789] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224166194524, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:16:06.195808] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224171195807, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:06.201499] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9DD-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.217587] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB5-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:06.220086] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB5-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:06.256336] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B9-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:06.258861] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B9-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:06.281850] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BE5-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}) [2025-08-15 10:16:06.281894] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BE5-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:16:06.285727] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=9] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:06.285769] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=30] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:06.285795] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=20] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:06.288256] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=16] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:06.371068] INFO [RS] prepare (ob_mview_maintenance_task.cpp:176) [727956][T1018_TimerWK0_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=14] mview maintenance task prepare success(tenant_id_=1018, round_=63408, prepare_cost_us_=0) [2025-08-15 10:16:06.402448] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=16][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227766401246}) [2025-08-15 10:16:06.402465] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:06.402475] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:06.408468] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227766407254}) [2025-08-15 10:16:06.408484] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:06.408497] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:06.409694] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227766408511}) [2025-08-15 10:16:06.409709] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:06.409721] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:06.431898] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF80-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.432200] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A28-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.434424] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF80-0-0] [lt=70] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:06.434442] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF80-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.434735] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A28-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:06.434753] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A28-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.442804] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6887-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.461840] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:06.461853] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:06.461858] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:06.502597] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0CE-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.502597] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E73D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.510414] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF222-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.533833] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:06.533852] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:06.533858] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:06.533866] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1016, cost_time_us=208, return_code_array=[0, 0, 0], broadcast_times=2) [2025-08-15 10:16:06.534392] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:06.534403] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:06.534411] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:06.538110] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAA6-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84570312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84960938GB}) [2025-08-15 10:16:06.538148] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAA6-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=40, ddl_arg=NULL) [2025-08-15 10:16:06.539678] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224164361748002, replayable_scn:1755224164361748002, readable_scn:1755224164361748002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224166536742, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224164361748002, replayable_scn:1755224164361748002, readable_scn:1755224164361748002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224166536742, ora_rowscn_=1755224165546322000, content_changed=true) [2025-08-15 10:16:06.539852] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:06.539868] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:06.539876] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:06.545957] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:06.545970] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:06.545976] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:06.550625] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FB-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.551263] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:06.551274] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:06.551279] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:06.553917] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FB-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.556027] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FB-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.556082] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C30-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.556132] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7869-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.556144] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FB-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.556325] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DA-0-0] [lt=5] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224166364390002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224166364390002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224166364390002, v:0}}], last_dump_ts_=1755224161525253, force_dump=false) [2025-08-15 10:16:06.556351] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DA-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.556425] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC062-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.559129] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C30-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.559127] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7869-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.559273] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DA-0-0] [lt=7] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224166364390002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224166364390002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224166364390002, v:0}}], last_dump_ts_=1755224161528206, force_dump=false) [2025-08-15 10:16:06.559304] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DA-0-0] [lt=23] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.559314] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC062-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224166364390002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224166364390002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224166364390002, v:0}}], last_dump_ts_=1755224161526429, force_dump=false) [2025-08-15 10:16:06.559337] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC062-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.561122] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7869-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.561208] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7869-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.561246] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C30-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.561287] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC062-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.561300] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DA-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.561338] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C30-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.561368] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC062-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.561388] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DA-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.624422] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8863-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.624929] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7659-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.627048] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8863-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:06.627066] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8863-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.627890] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7659-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.628586] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A42-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.629851] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7659-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.629954] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7659-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.631127] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A42-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:06.631145] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A42-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.691650] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C688A-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.703032] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD9C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.709090] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:06.709108] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:06.709118] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:06.752003] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDAB-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.752130] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF075-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.783631] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=16] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:06.814511] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709637][T1013_DRService][T1013][YB420ACB0122-000633A341D55698-0-0] [lt=12] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1013, thread_idx=1) [2025-08-15 10:16:06.818194] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [709637][T1013_DRService][T1013][YB420ACB0122-000633A341D55699-0-0] [lt=13] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2396, tenant_id=1013, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:06.818355] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=8] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:06.820550] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [709637][T1013_DRService][T1013][YB420ACB0122-000633A341D55699-0-0] [lt=13] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2337, tenant_id=1014, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:06.833647] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:06.834878] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224166833605, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:16:06.834902] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224171834901, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:16:06.849735] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D2B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.902650] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BC-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.905271] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BC-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:06.905289] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BC-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.922306] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8888-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.924843] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8888-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:06.924862] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8888-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.927167] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224166536742, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224164361748002, replayable_scn:1755224164361748002, readable_scn:1755224164361748002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:06.937971] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38DA-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:06.940484] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38DA-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:06.943176] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B385-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:06.955026] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4F-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.957638] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4F-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:06.957655] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB4F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:06.977275] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.980541] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.982627] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.982723] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:06.993122] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58F6-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.993846] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9D9-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:06.995082] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [709633][T1013_BackupDat][T1013][YB420ACB0122-000633A33E8598BE-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:06.996400] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [709632][T1013_BACKUP_SC][T1013][YB420ACB0122-000633A33E72C1CC-0-0] [lt=15] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:16:07.061213] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8FC-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:07.063714] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8FC-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:07.064981] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5271-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:07.067454] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5271-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:07.105577] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58F0-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.137867] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0D1-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.150238] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF072-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.153671] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727134][T1016_TimerWK2_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:07.156114] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A7-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:07.158245] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5310-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:07.159773] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A7-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:07.160819] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5310-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:07.187045] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D29-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.201573] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9DE-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.209351] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:07.225402] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB6-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:07.227922] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:07.263649] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=71] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:07.264231] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62BA-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:07.264603] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=9] mview_mds_op(tx_count=0) [2025-08-15 10:16:07.264615] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=8] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=862) [2025-08-15 10:16:07.266739] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62BA-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:07.323969] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487605][T1004_TimerWK2_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:07.338551] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:07.339805] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224167338478, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:16:07.339829] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224172339828, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:16:07.424819] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:07.426038] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224167424741, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:16:07.426061] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224172426060, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:16:07.437069] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E62-0-0] [lt=60] get number(ret=0, service_type_=0, range=1, base_id=1755224167436878000, start_id=1755224167436878000, end_id=1755224167436878001) [2025-08-15 10:16:07.437201] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF81-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.437434] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A29-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.437718] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E62-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224167436878000, v:0}, tenant_id=1017) [2025-08-15 10:16:07.439739] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF81-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:07.439755] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF81-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.440005] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A29-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:07.440021] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A29-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.442881] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6888-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.461835] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:07.461849] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:07.461856] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:07.470852] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A0932-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}) [2025-08-15 10:16:07.470890] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A0932-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:16:07.502673] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0CF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.502674] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E73E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.533834] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:07.533847] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:07.533853] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:07.534398] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:07.534409] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:07.534417] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:07.535701] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:07.539787] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:07.539798] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:07.539803] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:07.546035] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:07.546046] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:07.546052] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:07.547384] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:07.551323] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:07.551341] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:07.551350] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:07.557084] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FC-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.560061] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FC-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.562046] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FC-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.562158] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC063-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224167365380001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224167365380002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224167365380002, v:0}}], last_dump_ts_=1755224162529532, force_dump=false) [2025-08-15 10:16:07.562175] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.562184] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC063-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.562209] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FC-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.562299] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C31-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.562416] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DB-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.565058] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC063-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.565084] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786A-0-0] [lt=15] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224167365380003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224167365380003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224167365380003, v:0}}], last_dump_ts_=1755224162534134, force_dump=false) [2025-08-15 10:16:07.565106] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786A-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.565386] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C31-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.565393] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DB-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.567017] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786A-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.567092] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC063-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.567143] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.567183] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC063-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.567300] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DB-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.567305] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C31-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.567376] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DB-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.567388] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C31-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.571613] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E6B-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224167571521000, start_id=1755224167571521000, end_id=1755224167571521001) [2025-08-15 10:16:07.572249] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E6B-0-0] [lt=13] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224167571521000, v:0}, tenant_id=1015) [2025-08-15 10:16:07.572841] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66D-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224167572575000, start_id=1755224167572575000, end_id=1755224167572575001) [2025-08-15 10:16:07.573461] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66D-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224167572575000, v:0}, tenant_id=1005) [2025-08-15 10:16:07.629765] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8864-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.630904] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.632964] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8864-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:07.632984] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8864-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.633863] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A43-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.634402] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.636410] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.636460] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A43-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:07.636489] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.636479] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A43-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.691725] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C688B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.703108] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD9D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.708036] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:07.708052] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:07.708057] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:07.752061] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDAC-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.752201] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF076-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.782937] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC649-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.801025] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36CB-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224167800723000, start_id=1755224167800723000, end_id=1755224167800723001) [2025-08-15 10:16:07.801654] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36CB-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224167800723000, v:0}, tenant_id=1011) [2025-08-15 10:16:07.803355] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:07.804622] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224167803275, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:16:07.804649] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224172804648, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:16:07.805813] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224167804538, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:16:07.805832] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224172805831, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:16:07.849805] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D2C-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.908051] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BD-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.910660] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BD-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:07.910678] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BD-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.919106] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB3-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224167918510000, start_id=1755224167918510000, end_id=1755224167918510001) [2025-08-15 10:16:07.919761] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB3-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224167918510000, v:0}, tenant_id=1013) [2025-08-15 10:16:07.926712] INFO [RS] gc_mview (ob_mview_maintenance_task.cpp:267) [727133][T1016_TimerWK1_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=14] mview maintenance task gc mview success(tenant_id_=1016, round_=63409, gc_mview_cost_us_=1245, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:16:07.927568] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8889-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.930110] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8889-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:07.930126] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8889-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.944574] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B386-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:07.945783] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38DB-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:07.948269] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38DB-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:07.960419] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB50-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.962956] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB50-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:07.962972] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB50-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:07.966701] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:07.967952] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224167966624, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:16:07.967976] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224172967974, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:16:07.983711] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.986864] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.988813] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838B-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.988896] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:07.993197] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58F7-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:07.993922] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9DA-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.052415] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:08.053661] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224168052339, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:16:08.053688] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224173053686, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:16:08.069098] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8FD-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:08.072821] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8FD-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:08.072848] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5272-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:08.075370] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5272-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:08.098597] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B3-0-0] [lt=16] get number(ret=0, service_type_=0, range=1, base_id=1755224168098570000, start_id=1755224168098570000, end_id=1755224168098570001) [2025-08-15 10:16:08.099428] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B3-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224168098570000, v:0}, tenant_id=1012) [2025-08-15 10:16:08.105653] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58F1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.118324] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463BB-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224168117519000, start_id=1755224168117519000, end_id=1755224168117519001) [2025-08-15 10:16:08.119127] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463BB-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224168117519000, v:0}, tenant_id=1) [2025-08-15 10:16:08.137946] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0D2-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.150311] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF073-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.165137] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A8-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:08.166127] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5311-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:08.167675] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A8-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:08.168588] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5311-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:08.187118] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D2A-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.201644] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9DF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.233261] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB7-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:08.235718] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB7-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:08.260739] INFO [RS] recover_task (ob_ddl_scheduler.cpp:2771) [17349][TimerWK2_DDLScanTask][T0][Y0-0000000000000000-0-0] [lt=14] start processing ddl recovery(ddl_event_info={addr_:"10.203.1.34:2882", event_ts_:1755224168260738, sub_id_:0, trace_id_:Y0-0000000000000000-0-0, parent_trace_id_:Y0-0000000000000000-0-0}, task_records=[], primary_tenant_ids=[1004, 1006, 1008, 1012, 1014, 1016, 1018]) [2025-08-15 10:16:08.272078] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62BB-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:08.274582] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62BB-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:08.282390] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D62-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}) [2025-08-15 10:16:08.282435] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D62-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:16:08.442503] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF82-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.442711] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2A-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.442952] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6889-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.445056] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF82-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:08.445075] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF82-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.445330] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2A-0-0] [lt=70] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:08.445353] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2A-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.461753] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224166864929002, replayable_scn:1755224166864929002, readable_scn:1755224166864929002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224168458410, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224166864929002, replayable_scn:1755224166864929002, readable_scn:1755224166864929002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224168458410, ora_rowscn_=1755224167985904000, content_changed=true) [2025-08-15 10:16:08.461932] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:08.461951] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:08.461956] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:08.470609] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=13] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1005, thread_idx=0) [2025-08-15 10:16:08.471888] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1005, only_for_display=false) [2025-08-15 10:16:08.473146] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:08.487170] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.488334] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.489488] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.489510] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1005, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:08.489527] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.489538] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1005, ls_id:{id:1}, replicas:[{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:08.489642] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=81] build ls locality stat map(tenant_id=1005, ls_id={id:1}) [2025-08-15 10:16:08.489651] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.489663] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.489674] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.489686] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:08.489763] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.489777] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.489787] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.489793] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.489801] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3941, tenant_id=1005, ls_id={id:1}) [2025-08-15 10:16:08.489811] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:08.490876] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18989, tenant_id=1005, acc_dr_task=0) [2025-08-15 10:16:08.492124] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1006, only_for_display=false) [2025-08-15 10:16:08.493324] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:08.502727] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0D0-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.502747] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E73F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.507082] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.508244] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.509403] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.509426] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:08.509443] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.509453] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1}, replicas:[{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:08.509558] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=82] build ls locality stat map(tenant_id=1006, ls_id={id:1}) [2025-08-15 10:16:08.509566] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.509580] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.509590] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.509603] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:08.509680] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.509694] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.509703] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.509709] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.509718] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3808, tenant_id=1006, ls_id={id:1}) [2025-08-15 10:16:08.509729] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:08.510671] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF223-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.510859] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:08.531630] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.532784] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.533672] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224166524098000, replayable_scn:1755224166524098000, readable_scn:1755224166524098000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224168531052, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224166524098000, replayable_scn:1755224166524098000, readable_scn:1755224166524098000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224168531052, ora_rowscn_=1755224167633542000, content_changed=true) [2025-08-15 10:16:08.533860] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:08.533872] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:08.533882] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:08.533954] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.533977] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:08.533995] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.534009] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1001}, replicas:[{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:08.534114] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=82] build ls locality stat map(tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:16:08.534122] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.534136] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.534143] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.534154] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:08.534232] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.534246] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.534255] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.534262] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.534271] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3762, tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:16:08.534282] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:08.534317] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224166364390002, replayable_scn:1755224166364390002, readable_scn:1755224166364390002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224168531052, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224166364390002, replayable_scn:1755224166364390002, readable_scn:1755224166364390002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224168531052, ora_rowscn_=1755224167564147000, content_changed=true) [2025-08-15 10:16:08.534485] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:08.534497] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:08.534503] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:08.535364] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:08.538638] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D129F-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84960938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.85351562GB}) [2025-08-15 10:16:08.538681] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D129F-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:16:08.539881] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:08.539892] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:08.539898] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:08.545873] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224166364390002, replayable_scn:1755224166364390002, readable_scn:1755224166364390002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224168542599, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224166364390002, replayable_scn:1755224166364390002, readable_scn:1755224166364390002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224168542599, ora_rowscn_=1755224167564147000, content_changed=true) [2025-08-15 10:16:08.546054] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:08.546066] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:08.546071] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:08.551249] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224166364390002, replayable_scn:1755224166364390002, readable_scn:1755224166364390002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224168547867, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224166364390002, replayable_scn:1755224166364390002, readable_scn:1755224166364390002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224168547867, ora_rowscn_=1755224167564147000, content_changed=true) [2025-08-15 10:16:08.551432] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:08.551455] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:08.551463] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:08.556089] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.557254] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.558395] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.558418] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:08.558431] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.558442] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1002}, replicas:[{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:08.558545] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=80] build ls locality stat map(tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:16:08.558552] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.558564] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.558576] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.558587] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:08.558667] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=59] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.558681] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.558691] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.558699] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.558708] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3760, tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:16:08.558715] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:08.559818] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:08.563191] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FD-0-0] [lt=11] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224168366839001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224168366839001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224168366839001, v:0}}], last_dump_ts_=1755224163532781, force_dump=false) [2025-08-15 10:16:08.563217] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FD-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.566379] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FD-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224168415284001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224168415284001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224168415284001, v:0}}], last_dump_ts_=1755224163535191, force_dump=false) [2025-08-15 10:16:08.566399] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FD-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.568190] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC064-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.568209] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.568273] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FD-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.568297] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DC-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.568339] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C32-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.568353] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FD-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.571307] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786B-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.571343] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C32-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.571504] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DC-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.571580] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC064-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.573278] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C32-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.573366] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.573492] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C32-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.573563] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.573584] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC064-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.573639] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DC-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.573676] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC064-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.573728] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DC-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.580600] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.581742] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.582894] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.582920] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:08.582944] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.582953] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1003}, replicas:[{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:08.583075] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=95] build ls locality stat map(tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:16:08.583083] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.583094] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.583102] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.583114] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:08.583205] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=66] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.583217] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.583225] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.583231] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.583238] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3777, tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:16:08.583246] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:08.584377] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93486, tenant_id=1006, acc_dr_task=0) [2025-08-15 10:16:08.610476] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:08.611732] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224168610403, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:16:08.611758] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224173611756, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:16:08.635713] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8865-0-0] [lt=41] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.637496] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.638812] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8865-0-0] [lt=86] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:08.638829] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8865-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.639141] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A44-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.640428] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765B-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.641730] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A44-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:08.641748] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A44-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.642326] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.642429] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.652923] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=12] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1, thread_idx=0) [2025-08-15 10:16:08.654221] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=20] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-08-15 10:16:08.655413] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:08.669209] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.670357] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.671511] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:08.671551] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=31] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:08.671568] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.671580] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:08.671683] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=81] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-08-15 10:16:08.671692] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.671705] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.671716] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:08.671723] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:08.671799] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.671813] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.671822] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.671827] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:08.671837] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3777, tenant_id=1, ls_id={id:1}) [2025-08-15 10:16:08.671845] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:08.672935] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18714, tenant_id=1, acc_dr_task=0) [2025-08-15 10:16:08.691802] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C688C-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.703187] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD9E-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.707921] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224167365380003, replayable_scn:1755224167365380003, readable_scn:1755224167365380003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224168704652, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224167365380003, replayable_scn:1755224167365380003, readable_scn:1755224167365380003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224168704652, ora_rowscn_=1755224168569933000, content_changed=true) [2025-08-15 10:16:08.708113] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=24] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:08.708123] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:08.708130] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:08.752137] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDAD-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.752260] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF077-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.754535] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5B-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224168754081000, start_id=1755224168754081000, end_id=1755224168754081001) [2025-08-15 10:16:08.755136] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5B-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224168754081000, v:0}, tenant_id=1016) [2025-08-15 10:16:08.797805] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [438124][T1005_BackupDat][T1005][YB420ACB0122-0006338207177DA4-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:08.807519] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [438123][T1005_BACKUP_SC][T1005][YB420ACB0122-000633820702C25B-0-0] [lt=13] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:16:08.849880] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D2D-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.913366] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BE-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.913507] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619613][T1011_DRService][T1011][YB420ACB0122-0006338CC425E901-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1011, thread_idx=1) [2025-08-15 10:16:08.915900] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BE-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:08.915919] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BE-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.917121] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [2619613][T1011_DRService][T1011][YB420ACB0122-0006338CC425E902-0-0] [lt=13] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2362, tenant_id=1011, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:08.919465] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [2619613][T1011_DRService][T1011][YB420ACB0122-0006338CC425E902-0-0] [lt=19] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2319, tenant_id=1012, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:08.932851] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888A-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.935450] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888A-0-0] [lt=74] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:08.935468] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.945980] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B387-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:08.953615] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38DC-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:08.956083] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38DC-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:08.965727] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB51-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.968275] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB51-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:08.968294] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB51-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:08.989972] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.993131] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838C-0-0] [lt=9] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224168867667001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224168867667002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224168867667002, v:0}}], last_dump_ts_=1755224163961009, force_dump=false) [2025-08-15 10:16:08.993156] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.993270] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58F8-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.994002] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9DB-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:08.995122] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:08.995312] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.078290] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8FE-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:09.080652] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5273-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:09.080813] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8FE-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:09.083147] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5273-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:09.105729] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58F2-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.138011] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0D3-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.150390] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF074-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.170468] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64E-0-0] [lt=60] get number(ret=0, service_type_=0, range=1, base_id=1755224169169527000, start_id=1755224169169527000, end_id=1755224169169527001) [2025-08-15 10:16:09.171260] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64E-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224169169527000, v:0}, tenant_id=1006) [2025-08-15 10:16:09.173064] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A9-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:09.173864] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5312-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:09.175553] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86A9-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:09.176415] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5312-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:09.187190] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D2B-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.201719] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9E0-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.234600] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A7-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224169233740000, start_id=1755224169233740000, end_id=1755224169233740001) [2025-08-15 10:16:09.235383] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A7-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224169233740000, v:0}, tenant_id=1008) [2025-08-15 10:16:09.241048] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB8-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:09.243512] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB8-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:09.279937] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62BC-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:09.282446] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62BC-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:09.413988] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=21][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227769412801}) [2025-08-15 10:16:09.414004] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:09.414012] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:09.415798] INFO [RS] prepare (ob_mview_maintenance_task.cpp:176) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=43] mview maintenance task prepare success(tenant_id_=1, round_=121920, prepare_cost_us_=0) [2025-08-15 10:16:09.419911] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227769418740}) [2025-08-15 10:16:09.419924] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:09.419934] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:09.421117] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227769419950}) [2025-08-15 10:16:09.421129] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:09.421139] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:09.442466] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1003, thread_idx=0) [2025-08-15 10:16:09.443003] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C688A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.443733] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1003, only_for_display=false) [2025-08-15 10:16:09.444989] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:09.447834] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF83-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.448100] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2B-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.450372] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF83-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:09.450391] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF83-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.450644] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2B-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:09.450663] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.458949] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.460168] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=26] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.461338] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.461371] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1003, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:09.461391] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=13] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.461403] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1003, ls_id:{id:1}, replicas:[{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:09.461520] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=91] build ls locality stat map(tenant_id=1003, ls_id={id:1}) [2025-08-15 10:16:09.461532] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.461548] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=12] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.461561] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=10] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.461572] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:09.461652] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.461668] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.461677] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.461683] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.461692] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3911, tenant_id=1003, ls_id={id:1}) [2025-08-15 10:16:09.461702] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:09.461929] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:09.461945] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:09.461953] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:09.462781] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19048, tenant_id=1003, acc_dr_task=0) [2025-08-15 10:16:09.463231] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:09.464023] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1004, only_for_display=false) [2025-08-15 10:16:09.465205] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:09.471371] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E420-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}) [2025-08-15 10:16:09.471409] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E420-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:16:09.478901] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.480066] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.481210] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.481233] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:09.481250] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.481262] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:09.481366] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=82] build ls locality stat map(tenant_id=1004, ls_id={id:1}) [2025-08-15 10:16:09.481375] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.481389] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.481400] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.481412] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:09.481488] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.481502] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.481512] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.481521] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.481529] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3766, tenant_id=1004, ls_id={id:1}) [2025-08-15 10:16:09.481539] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:09.482598] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:09.502787] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0D1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.502825] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E740-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.503553] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.504722] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.505866] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.505890] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:09.505908] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.505919] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:09.506032] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=89] build ls locality stat map(tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:16:09.506040] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.506054] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.506064] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.506075] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:09.506152] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.506166] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.506176] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.506181] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.506190] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3782, tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:16:09.506201] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:09.507258] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:09.528084] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.529259] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.530422] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.530445] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:09.530461] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.530472] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1002}, replicas:[{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:09.530597] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=97] build ls locality stat map(tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:16:09.530605] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.530620] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.530637] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=12] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.530649] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:09.530727] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.530742] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.530752] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.530761] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.530766] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3835, tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:16:09.530777] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:09.531840] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:09.533959] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:09.533974] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:09.533982] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:09.533994] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1016, cost_time_us=199, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:09.534545] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:09.534557] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:09.534564] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:09.535303] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:09.539763] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224167365380004, replayable_scn:1755224167365380004, readable_scn:1755224167365380004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224169536781, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224167365380004, replayable_scn:1755224167365380004, readable_scn:1755224167365380004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224169536781, ora_rowscn_=1755224168564687000, content_changed=true) [2025-08-15 10:16:09.539979] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:09.539996] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:09.540003] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:09.546109] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:09.546121] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:09.546127] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:09.551432] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:09.551444] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:09.551450] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:09.552737] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:09.552857] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.554034] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.555188] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.555212] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:09.555230] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.555241] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1003}, replicas:[{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:09.555347] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=82] build ls locality stat map(tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:16:09.555354] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.555368] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.555379] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.555390] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:09.555467] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.555480] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.555489] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.555496] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.555505] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3798, tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:16:09.555515] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:09.556594] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93799, tenant_id=1004, acc_dr_task=0) [2025-08-15 10:16:09.569376] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FE-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.572634] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FE-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.574439] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C33-0-0] [lt=9] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224169368485001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224169368485002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224169368485002, v:0}}], last_dump_ts_=1755224164544204, force_dump=false) [2025-08-15 10:16:09.574465] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C33-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.574522] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786C-0-0] [lt=4] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224169368485002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224169368485002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224169368485002, v:0}}], last_dump_ts_=1755224164543781, force_dump=false) [2025-08-15 10:16:09.574543] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786C-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.574615] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FE-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.574622] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC065-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.574673] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DD-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.574693] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FE-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.577463] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786C-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.577487] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C33-0-0] [lt=17] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224169368485001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224169368485002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224169368485002, v:0}}], last_dump_ts_=1755224164547140, force_dump=false) [2025-08-15 10:16:09.577512] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C33-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.577746] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC065-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.577773] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DD-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.579457] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C33-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.579459] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786C-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.579536] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C33-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.579546] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786C-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.579755] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC065-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.579833] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC065-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.579946] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DD-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.580070] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DD-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.631088] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:09.632321] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224169631013, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:16:09.632345] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224174632343, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:16:09.641555] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8866-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.643408] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765C-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224169368485002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224169529406000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224169529406000, v:0}}], last_dump_ts_=1755224164612901, force_dump=false) [2025-08-15 10:16:09.643433] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.644098] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8866-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:09.644116] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8866-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.644415] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A45-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.646338] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765C-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224169529406000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224169529406000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224169529406000, v:0}}], last_dump_ts_=1755224164615890, force_dump=false) [2025-08-15 10:16:09.646362] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765C-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.647029] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A45-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:09.647046] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A45-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.648266] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.648420] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.691875] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C688D-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.703238] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD9F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.708147] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:09.708160] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:09.708167] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:09.709439] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:09.728983] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E7-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224169728217000, start_id=1755224169728217000, end_id=1755224169728217001) [2025-08-15 10:16:09.729675] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E7-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224169728217000, v:0}, tenant_id=1003) [2025-08-15 10:16:09.752211] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDAE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.752335] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF078-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.778794] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1017, thread_idx=0) [2025-08-15 10:16:09.780046] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1017, only_for_display=false) [2025-08-15 10:16:09.781263] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:09.787873] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC64A-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.795190] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.796338] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.797481] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.797504] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1017, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:09.797518] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.797527] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1017, ls_id:{id:1}, replicas:[{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:09.797629] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=79] build ls locality stat map(tenant_id=1017, ls_id={id:1}) [2025-08-15 10:16:09.797639] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.797650] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.797658] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.797667] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:09.797741] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.797753] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.797760] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.797766] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.797773] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3737, tenant_id=1017, ls_id={id:1}) [2025-08-15 10:16:09.797781] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:09.798833] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18787, tenant_id=1017, acc_dr_task=0) [2025-08-15 10:16:09.800052] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1018, only_for_display=false) [2025-08-15 10:16:09.801260] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:09.814967] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.816113] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.817264] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.817286] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:09.817304] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.817317] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1}, replicas:[{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:09.817420] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=81] build ls locality stat map(tenant_id=1018, ls_id={id:1}) [2025-08-15 10:16:09.817428] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.817441] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.817452] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.817463] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:09.817540] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.817550] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.817559] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.817564] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.817573] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3766, tenant_id=1018, ls_id={id:1}) [2025-08-15 10:16:09.817583] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:09.818646] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:09.839403] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.840554] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.841708] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.841731] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:09.841744] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.841756] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1001}, replicas:[{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:09.841859] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:16:09.841867] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.841881] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.841890] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.841901] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:09.841991] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=67] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.842007] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.842017] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.842022] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.842031] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3770, tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:16:09.842041] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:09.843124] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:09.849953] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D2E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.863971] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.865117] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.866253] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.866275] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:09.866292] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.866303] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1002}, replicas:[{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:09.866408] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=82] build ls locality stat map(tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:16:09.866417] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.866430] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.866441] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.866452] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:09.866529] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.866543] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.866552] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.866558] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.866567] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3747, tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:16:09.866576] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:09.867640] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:09.869965] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C4-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224169869212000, start_id=1755224169869212002, end_id=1755224169869212003) [2025-08-15 10:16:09.870416] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1080) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C4-0-0] [lt=15] [PALF STAT GROUP LOG INFO](palf_id=1, self="10.203.1.34:2882", role="LEADER", total_group_log_cnt=11, avg_log_batch_cnt=1, total_group_log_size=2457, avg_group_log_size=223) [2025-08-15 10:16:09.870433] INFO [PALF] submit_log (palf_handle_impl.cpp:446) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C4-0-0] [lt=12] [PALF STAT APPEND DATA SIZE](this={palf_id:1, self:"10.203.1.34:2882", has_set_deleted:false}, append size=1765) [2025-08-15 10:16:09.870753] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C4-0-0] [lt=9] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224169869212002, v:0}, tenant_id=1004) [2025-08-15 10:16:09.870771] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C4-0-0] [lt=12] zone_merge_mgr is already loaded(tenant_id=1004, global_merge_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402914553, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194735253569, need_update:false}}, zone_merge_infos=[{tenant_id:1004, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194735255910, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403065502, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1004, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194735259562, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403073022, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1004, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194735261458, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403077657, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:16:09.888584] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.889725] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.890873] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:09.890895] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:09.890910] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.890920] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1003}, replicas:[{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:09.891029] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=86] build ls locality stat map(tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:16:09.891037] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.891048] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.891056] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:09.891063] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:09.891145] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.891157] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.891164] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.891171] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:09.891177] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3751, tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:16:09.891185] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:09.892261] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93417, tenant_id=1018, acc_dr_task=0) [2025-08-15 10:16:09.918580] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BF-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.921098] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BF-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:09.921115] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95BF-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.928732] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224169536781, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224167365380004, replayable_scn:1755224167365380004, readable_scn:1755224167365380004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:09.938100] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888B-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.940612] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888B-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:09.940629] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888B-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.947359] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B388-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:09.961402] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38DD-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:09.963879] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38DD-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:09.971060] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB52-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.973586] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB52-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:09.973606] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB52-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:09.993344] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58F9-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.994063] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9DC-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:09.996318] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838D-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224169869212001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224169869212002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224169869212002, v:0}}], last_dump_ts_=1755224164964213, force_dump=false) [2025-08-15 10:16:09.996342] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838D-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:09.999364] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.001327] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.001403] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.008127] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=10] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:10.086344] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8FF-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:10.088463] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5274-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:10.088864] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8FF-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:10.090912] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5274-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:10.105805] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58F3-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.138083] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0D4-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.150459] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF075-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.180821] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86AA-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:10.181649] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5313-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:10.183362] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86AA-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:10.184125] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5313-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:10.187270] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D2C-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.200832] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C5-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224170200723000, start_id=1755224170200723000, end_id=1755224170200723001) [2025-08-15 10:16:10.201638] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C5-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224170200723000, v:0}, tenant_id=1007) [2025-08-15 10:16:10.201789] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9E1-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.211319] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727239][T1015_DRService][T1015][YB420ACB0122-000633A351E5562D-0-0] [lt=16] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1015, thread_idx=1) [2025-08-15 10:16:10.214994] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [727239][T1015_DRService][T1015][YB420ACB0122-000633A351E5562E-0-0] [lt=13] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2388, tenant_id=1015, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:10.217338] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [727239][T1015_DRService][T1015][YB420ACB0122-000633A351E5562E-0-0] [lt=14] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2323, tenant_id=1016, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:10.248802] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB9-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:10.251268] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FB9-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:10.282904] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B636D-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}) [2025-08-15 10:16:10.282953] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B636D-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=54, ddl_arg=NULL) [2025-08-15 10:16:10.287835] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62BD-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:10.290338] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62BD-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:10.323822] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [438024][T1006_TimerWK3_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=10] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:10.323859] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [438024][T1006_TimerWK3_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=25] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:10.323874] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [438024][T1006_TimerWK3_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=10] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:10.325093] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [438023][T1006_TimerWK2_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=10] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:10.339608] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [17712][T1_BackupDataSr][T1][YB420ACB0122-0006337F03F7B810-0-0] [lt=16] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:10.443076] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C688B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.453134] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF84-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.453355] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2C-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.455686] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF84-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:10.455704] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF84-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.455981] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2C-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:10.456000] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2C-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.462069] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:10.462082] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:10.462087] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:10.502858] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0D2-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.502903] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E741-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.510959] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF224-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.534579] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:10.534584] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:10.534591] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:10.534594] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:10.534596] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:10.534600] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:10.539134] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D12AA-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.85351562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.85742188GB}) [2025-08-15 10:16:10.539179] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D12AA-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:16:10.540514] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:10.540526] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:10.540534] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:10.541801] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:10.546171] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:10.546188] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:10.546197] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:10.551452] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:10.551463] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:10.551469] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:10.575665] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FF-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.578777] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FF-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.580482] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C34-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.580485] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.580756] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC066-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.580913] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FF-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.581048] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76FF-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.581086] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DE-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.583520] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.583528] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C34-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.583789] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC066-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.584241] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DE-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.585502] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C34-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.585555] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.585589] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C34-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.585637] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.585791] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC066-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.585873] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC066-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.586205] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DE-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.586301] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DE-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.599889] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17716][T1_DRService][T1][YB420ACB0122-0006337F0497B7A1-0-0] [lt=15] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1, thread_idx=1) [2025-08-15 10:16:10.603418] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [17716][T1_DRService][T1][YB420ACB0122-0006337F0497B7A2-0-0] [lt=17] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2289, tenant_id=1, need_clean_task=true, service_epoch_to_check=2) [2025-08-15 10:16:10.626109] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:10.627358] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224170626036, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:16:10.627381] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224175627379, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:16:10.647044] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8867-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.649388] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.649579] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8867-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:10.649597] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8867-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.649762] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A46-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.652385] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A46-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:10.652392] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765D-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.652402] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A46-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.654301] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.654386] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:10.691947] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C688E-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.703307] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDA0-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.708223] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:10.708235] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:10.708241] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:10.717506] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [3414128][T1006_TimerWK4_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=17] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:10.752284] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDAF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.752406] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF079-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.850024] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D2F-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.923871] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C0-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.926387] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C0-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:10.926404] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C0-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.943300] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888C-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.945796] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888C-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:10.945812] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888C-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.948741] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B389-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:10.950689] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:10.951985] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224170950613, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:16:10.952010] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224175952008, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:16:10.969169] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38DE-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:10.972813] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38DE-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:10.976323] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB53-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.978838] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB53-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:10.978856] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB53-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:10.993419] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58FA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:10.994122] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9DD-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.002373] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.005419] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.007286] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.007380] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.047549] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:11.048774] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224171038801, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:16:11.048799] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224176048797, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:16:11.048816] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224176048813, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:11.094215] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E900-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:11.096212] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5275-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:11.096758] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E900-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:11.098698] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5275-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:11.105881] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58F4-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.138159] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0D5-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.150531] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF076-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.187343] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D2D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.188740] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86AB-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:11.189403] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5314-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:11.191234] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86AB-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:11.191863] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5314-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:11.193589] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:11.194813] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224171193510, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:16:11.194837] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224176194835, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:16:11.195874] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:11.197080] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224171195807, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:16:11.197102] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224176197101, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:11.201864] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9E2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.256654] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FBA-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:11.259129] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FBA-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:11.296142] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62BE-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:11.298612] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62BE-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:11.413572] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E52-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224171412930000, start_id=1755224171412930000, end_id=1755224171412930001) [2025-08-15 10:16:11.414347] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E52-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224171412930000, v:0}, tenant_id=1018) [2025-08-15 10:16:11.443148] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C688C-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.458473] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF85-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.458706] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2D-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.461005] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF85-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:11.461025] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF85-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.461273] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2D-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:11.461290] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.461861] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224169869212002, replayable_scn:1755224169869212002, readable_scn:1755224169869212002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224171459186, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224169869212002, replayable_scn:1755224169869212002, readable_scn:1755224169869212002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224171459186, ora_rowscn_=1755224171003728000, content_changed=true) [2025-08-15 10:16:11.462047] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:11.462059] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:11.462065] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:11.471851] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D22-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}) [2025-08-15 10:16:11.471890] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D22-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:16:11.502933] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0D3-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.502976] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E742-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.533811] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224169529406000, replayable_scn:1755224169529406000, readable_scn:1755224169529406000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224171530790, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224169529406000, replayable_scn:1755224169529406000, readable_scn:1755224169529406000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224171530790, ora_rowscn_=1755224170650766000, content_changed=true) [2025-08-15 10:16:11.533991] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:11.534003] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:11.534010] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:11.534450] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224169368485002, replayable_scn:1755224169368485002, readable_scn:1755224169368485002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224171531844, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224169368485002, replayable_scn:1755224169368485002, readable_scn:1755224169368485002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224171531844, ora_rowscn_=1755224170582394000, content_changed=true) [2025-08-15 10:16:11.534610] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:11.534619] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:11.534626] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:11.540003] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:11.540014] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:11.540019] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:11.546074] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224169368485002, replayable_scn:1755224169368485002, readable_scn:1755224169368485002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224171543426, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224169368485002, replayable_scn:1755224169368485002, readable_scn:1755224169368485002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224171543426, ora_rowscn_=1755224170582394000, content_changed=true) [2025-08-15 10:16:11.546252] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:11.546263] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:11.546272] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:11.551307] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224169368485002, replayable_scn:1755224169368485002, readable_scn:1755224169368485002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224171548693, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224169368485002, replayable_scn:1755224169368485002, readable_scn:1755224169368485002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224171548693, ora_rowscn_=1755224170582394000, content_changed=true) [2025-08-15 10:16:11.551487] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:11.551504] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:11.551511] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:11.582037] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7700-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.585192] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7700-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.586459] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C35-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.586539] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.586805] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC067-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.587205] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7700-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.587214] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DF-0-0] [lt=5] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224171370863002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224171370863002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224171370863002, v:0}}], last_dump_ts_=1755224166556324, force_dump=false) [2025-08-15 10:16:11.587237] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DF-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.587302] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7700-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.589519] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.589601] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C35-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.589737] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC067-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224171370863001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224171370863002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224171370863002, v:0}}], last_dump_ts_=1755224166559314, force_dump=false) [2025-08-15 10:16:11.589764] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC067-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.590193] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DF-0-0] [lt=8] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224171370863002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224171370863002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224171370863001, v:0}}], last_dump_ts_=1755224166559272, force_dump=false) [2025-08-15 10:16:11.590216] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DF-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.591650] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.591732] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786E-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.591734] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C35-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.591827] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C35-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.591849] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC067-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.591932] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC067-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.592090] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DF-0-0] [lt=3] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.592175] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413DF-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.652344] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8868-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.654904] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8868-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:11.654920] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8868-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.655140] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A47-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.655309] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.657765] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A47-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:11.657781] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A47-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.658217] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.660118] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.660228] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:11.692023] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C688F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.703384] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDA1-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.708103] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224170370140001, replayable_scn:1755224170370140001, readable_scn:1755224170370140001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224171705494, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224170370140001, replayable_scn:1755224170370140001, readable_scn:1755224170370140001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224171705494, ora_rowscn_=1755224171588620000, content_changed=true) [2025-08-15 10:16:11.708285] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:11.708302] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:11.708314] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:11.729098] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D4-0-0] [lt=15] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1015, thread_idx=0) [2025-08-15 10:16:11.730415] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1015, only_for_display=false) [2025-08-15 10:16:11.731643] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:11.745472] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.746729] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.747930] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.747958] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1015, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:11.747971] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.747981] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1015, ls_id:{id:1}, replicas:[{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:11.748083] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=80] build ls locality stat map(tenant_id=1015, ls_id={id:1}) [2025-08-15 10:16:11.748090] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.748101] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.748107] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.748115] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:11.748191] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.748202] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.748209] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.748216] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.748223] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3897, tenant_id=1015, ls_id={id:1}) [2025-08-15 10:16:11.748230] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:11.749292] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18877, tenant_id=1015, acc_dr_task=0) [2025-08-15 10:16:11.750506] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1016, only_for_display=false) [2025-08-15 10:16:11.751732] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:11.752358] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDB0-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.752482] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF07A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.765662] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.766829] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.767989] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.768010] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:11.768022] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.768031] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1}, replicas:[{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:11.768131] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=79] build ls locality stat map(tenant_id=1016, ls_id={id:1}) [2025-08-15 10:16:11.768140] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.768152] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.768160] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.768169] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:11.768245] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=54] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.768257] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.768265] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.768272] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.768278] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3769, tenant_id=1016, ls_id={id:1}) [2025-08-15 10:16:11.768286] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:11.769356] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:11.784978] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709536][T1014_TimerWK3_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=16] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:11.790182] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.791326] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.792464] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.792487] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:11.792505] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.792517] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1001}, replicas:[{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:11.792619] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=80] build ls locality stat map(tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:16:11.792628] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.792641] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.792651] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.792709] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC64B-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.792663] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:11.792740] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.792753] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.792763] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.792768] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.792777] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3743, tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:16:11.792783] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:11.793930] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:11.807903] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA2-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224171807792000, start_id=1755224171807792000, end_id=1755224171807792001) [2025-08-15 10:16:11.808702] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA2-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224171807792000, v:0}, tenant_id=1014) [2025-08-15 10:16:11.815242] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.816414] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.817568] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.817591] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:11.817608] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.817619] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1002}, replicas:[{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:11.817728] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=86] build ls locality stat map(tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:16:11.817737] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.817749] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.817760] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.817772] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:11.817858] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=63] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.817873] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.817882] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.817887] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.817898] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=9] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3815, tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:16:11.817909] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:11.818989] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:11.824029] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:11.834974] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:11.836212] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224171834901, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:16:11.836235] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224176836234, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:16:11.840079] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.841239] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.842391] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:11.842415] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:11.842431] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.842442] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1003}, replicas:[{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:11.842546] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=81] build ls locality stat map(tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:16:11.842553] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.842566] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.842577] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:11.842587] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:11.842669] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.842682] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.842692] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.842697] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:11.842706] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3786, tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:16:11.842718] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:11.843805] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94500, tenant_id=1016, acc_dr_task=0) [2025-08-15 10:16:11.850095] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D30-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.929113] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C1-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.931628] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C1-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:11.931642] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C1-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.948486] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888D-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.950157] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B38A-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:11.951003] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888D-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:11.951021] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.978168] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38DF-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:11.980656] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38DF-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:11.981591] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB54-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.984178] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB54-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:11.984194] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB54-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:11.993494] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58FB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:11.994191] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9DE-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.008488] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838F-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.011503] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.013399] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.013481] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D838F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.102115] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E901-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:12.104065] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5276-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:12.104706] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E901-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:12.105956] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58F5-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.106566] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5276-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:12.138225] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0D6-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.150560] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF077-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.159040] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727133][T1016_TimerWK1_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=16] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:12.187414] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D2E-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.196509] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86AC-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:12.197090] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5315-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:12.198990] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86AC-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:12.199614] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5315-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:12.201902] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9E3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.210731] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727957][T1018_TimerWK1_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=17] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:12.264478] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FBB-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:12.264996] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619488][T1012_TimerWK3_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:12.267013] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FBB-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:12.283438] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D6E-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}) [2025-08-15 10:16:12.283481] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D6E-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=47, ddl_arg=NULL) [2025-08-15 10:16:12.303967] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62BF-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:12.306460] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62BF-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:12.325282] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487606][T1004_TimerWK3_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=16] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:12.339863] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:12.341103] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224172339828, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:16:12.341127] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224177341125, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:16:12.425408] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227772424213}) [2025-08-15 10:16:12.425424] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:12.425435] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:12.431403] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227772430230}) [2025-08-15 10:16:12.431417] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:12.431430] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:12.431466] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:12.432614] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227772431443}) [2025-08-15 10:16:12.432628] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:12.432637] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:12.432701] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224172426060, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:16:12.432725] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224177432724, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:16:12.443218] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C688D-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.462064] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:12.462085] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:12.462091] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:12.463853] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF86-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.464010] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2E-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.466386] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF86-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:12.466404] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF86-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.466532] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2E-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:12.466550] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2E-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.482688] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728063][T1017_DRService][T1017][YB420ACB0122-000633A358855627-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1017, thread_idx=1) [2025-08-15 10:16:12.486373] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [728063][T1017_DRService][T1017][YB420ACB0122-000633A358855628-0-0] [lt=13] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2388, tenant_id=1017, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:12.488712] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [728063][T1017_DRService][T1017][YB420ACB0122-000633A358855628-0-0] [lt=17] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2316, tenant_id=1018, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:12.503008] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0D4-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.503046] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E743-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.511242] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF225-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.534089] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:12.534101] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:12.534106] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:12.534682] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:12.534694] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:12.534703] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:12.534715] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1012, cost_time_us=187, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:12.535984] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:12.539599] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5830-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.85742188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80273438GB}) [2025-08-15 10:16:12.539644] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5830-0-0] [lt=29] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:16:12.539949] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=12] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224170370140002, replayable_scn:1755224170370140002, readable_scn:1755224170370140002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224172537764, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224170370140002, replayable_scn:1755224170370140002, readable_scn:1755224170370140002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224172537764, ora_rowscn_=1755224171583432000, content_changed=true) [2025-08-15 10:16:12.540118] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:12.540130] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:12.540135] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:12.546195] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:12.546210] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:12.546217] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:12.547561] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:12.551524] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:12.551535] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:12.551543] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:12.588294] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7701-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.591383] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7701-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.592727] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.592752] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C36-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.592860] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC068-0-0] [lt=16] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224172372569001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224172372569002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224172372569002, v:0}}], last_dump_ts_=1755224167562158, force_dump=false) [2025-08-15 10:16:12.592886] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC068-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.593073] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E0-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.593326] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7701-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.593410] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7701-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.595866] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786F-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224172372569002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224172372569002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224172372569002, v:0}}], last_dump_ts_=1755224167565083, force_dump=false) [2025-08-15 10:16:12.595886] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786F-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.595913] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC068-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.595951] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E0-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.595959] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C36-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.597878] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786F-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.597949] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC068-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.597987] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB786F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.597994] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E0-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.598062] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C36-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.598083] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC068-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.598148] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C36-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.598179] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E0-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.657652] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8869-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.660199] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8869-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:12.660216] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8869-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.660523] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A48-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.661232] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.663113] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A48-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:12.663136] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A48-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.664215] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765F-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.666093] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.666191] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D765F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:12.692097] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6890-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.703462] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDA2-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.708471] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:12.708483] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:12.708488] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:12.752428] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDB1-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.752554] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF07B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.804730] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:12.805988] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224172804648, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:16:12.806012] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224177806010, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:16:12.807201] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224172805831, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:16:12.807221] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224177807220, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:16:12.817144] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EB-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1013, thread_idx=0) [2025-08-15 10:16:12.818416] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=17] start try tenant disaster recovery(tenant_id=1013, only_for_display=false) [2025-08-15 10:16:12.819643] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:12.833756] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.834905] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.836073] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.836096] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1013, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:12.836114] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.836126] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1013, ls_id:{id:1}, replicas:[{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:12.836229] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=80] build ls locality stat map(tenant_id=1013, ls_id={id:1}) [2025-08-15 10:16:12.836238] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.836252] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.836262] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.836273] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:12.836350] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.836363] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.836373] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.836377] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.836385] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3784, tenant_id=1013, ls_id={id:1}) [2025-08-15 10:16:12.836395] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:12.837458] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19041, tenant_id=1013, acc_dr_task=0) [2025-08-15 10:16:12.838742] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1014, only_for_display=false) [2025-08-15 10:16:12.839959] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:12.850169] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D31-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.853859] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.855021] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.856243] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.856265] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:12.856280] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.856292] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1}, replicas:[{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:12.856393] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=80] build ls locality stat map(tenant_id=1014, ls_id={id:1}) [2025-08-15 10:16:12.856401] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.856414] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.856424] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.856433] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:12.856510] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.856524] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.856534] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.856539] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.856547] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3883, tenant_id=1014, ls_id={id:1}) [2025-08-15 10:16:12.856557] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:12.857643] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:12.878837] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.880003] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.881155] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.881179] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:12.881196] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.881208] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1001}, replicas:[{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:12.881320] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=88] build ls locality stat map(tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:16:12.881329] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.881342] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.881352] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.881364] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:12.881441] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.881455] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.881464] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.881469] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.881478] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3845, tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:16:12.881489] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:12.882549] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:12.903574] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.904760] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.905926] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.905955] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:12.905972] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.905984] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1002}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:12.906088] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=81] build ls locality stat map(tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:16:12.906097] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.906114] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=14] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.906125] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.906137] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:12.906233] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=72] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.906247] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.906256] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.906262] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.906271] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3843, tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:16:12.906279] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:12.907337] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:12.928343] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.929507] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.930402] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224172548246, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224170370140002, replayable_scn:1755224170370140002, readable_scn:1755224170370140001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:12.930657] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:12.930678] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:12.930695] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.930704] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1003}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:12.930806] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=80] build ls locality stat map(tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:16:12.930814] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.930825] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.930833] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:12.930841] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:12.930917] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.930929] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.930937] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.930949] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=10] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:12.930953] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3774, tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:16:12.930961] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:12.932039] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94567, tenant_id=1014, acc_dr_task=0) [2025-08-15 10:16:12.934364] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C2-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.936905] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C2-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:12.936921] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C2-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.951593] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B38B-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:12.953708] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=18] start do ObArchiveSchedulerService round(round=101220) [2025-08-15 10:16:12.953751] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.955618] INFO [ARCHIVE] checkpoint_ (ob_tenant_archive_scheduler.cpp:682) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=14] schedule archive(round_info={key:{tenant_id:1016, dest_no:0}, incarnation:1, dest_id:1001, round_id:1, state:{status:"DOING"}, start_scn:{val:1752091205985713000, v:0}, checkpoint_scn:{val:1755224005638556002, v:0}, max_scn:{val:1755224059891515001, v:0}, compatible:{version:1}, base_piece_id:1, used_piece_id:37, piece_switch_interval:86400000000, frozen_input_bytes:44838045649, frozen_output_bytes:44838045649, active_input_bytes:331266514, active_output_bytes:331266514, deleted_input_bytes:0, deleted_output_bytes:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com", comment:""}) [2025-08-15 10:16:12.957502] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888E-0-0] [lt=82] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:12.957523] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888E-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.957676] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727237][T1015_ArchiveSv][T1016][YB420ACB0122-000633A34F144D2F-0-0] [lt=28] get number(ret=0, service_type_=0, range=1, base_id=1755224172957308000, start_id=1755224172957308000, end_id=1755224172957308001) [2025-08-15 10:16:12.957693] INFO [SHARE] get_backup_scn (ob_backup_struct.cpp:2155) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=12] get tenant gts(ret=0, ret="OB_SUCCESS", tenant_id=1016, scn={val:1755224172957308000, v:0}) [2025-08-15 10:16:12.957712] INFO [SHARE] count_ (ob_archive_checkpoint.cpp:173) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=11] print count result(ret=0, counter={ls_count:4, deleted_ls_count:0, not_start_cnt:0, interrupted_cnt:0, doing_cnt:4, stopped_cnt:0, interrupted_ls_id:{id:-1}, max_scn:{val:1755224133103752001, v:0}, checkpoint_scn:{val:1755224058213484001, v:0}, max_active_piece_id:37}) [2025-08-15 10:16:12.958906] INFO [SHARE] get_backup_dest (ob_backup_connectivity.cpp:1268) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=10] success get backup dest(sql=SELECT authorization,extension FROM __all_backup_storage_info WHERE tenant_id = 1016 AND path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog' AND endpoint = 'host=cos.ap-beijing.myqcloud.com', tenant_id=1016, backup_dest={root_path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog", storage_info:{endpoint:"host=cos.ap-beijing.myqcloud.com", access_id:"access_id=AKIDiKl3A5ZXJ959EMuL3hJ1g9PTM5lFvjhT", extension:"appid=1254236265&checksum_type=md5", type:"COS", checksum_type:1, max_iops:0, max_bandwidth:0, role_arn:0x7f1bf3892f90, external_id:0x7f1bf3893090}}) [2025-08-15 10:16:12.968046] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:12.969295] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224172967974, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:16:12.969319] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224177969317, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:16:12.986043] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E0-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:12.986984] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB55-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.988538] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E0-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:12.989524] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB55-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:12.989539] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB55-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:12.993573] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58FC-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.994267] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9DF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:12.996498] INFO mk_parent_dir (ob_backup_io_adapter.cpp:346) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=16] found parent dir(i=130, path="cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog/piece_d1001r1p37/checkpoint") [2025-08-15 10:16:13.014463] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8390-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.017210] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:340) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8390-0-0] [lt=10] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1014, ls_id:{id:1}, sync_scn:{val:1755224172873175001, v:0}, readable_scn:{val:1755224172873175001, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-08-15 10:16:13.017603] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8390-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.019541] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8390-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.019620] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8390-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.019656] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:459) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8390-0-0] [lt=5] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1755224170870393002, v:0}) [2025-08-15 10:16:13.039062] INFO [STORAGE] print_access_storage_log (ob_storage.cpp:46) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=13] access storage op=write_single_file uri=cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog/piece_d1001r1p37/checkpoint/checkpoint_info.1755224058213484001.obarc size=0 Byte cost_ts=42537 us speed=0.00 MB/s [2025-08-15 10:16:13.039082] INFO [SHARE] write_checkpoint_file_ (ob_archive_checkpoint_mgr.cpp:255) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=16] succeed to write checkpoint file.(path={cur_pos:172, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog/piece_d1001r1p37/checkpoint/checkpoint_info.1755224058213484001.obarc"}) [2025-08-15 10:16:13.053761] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:13.054989] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224173053686, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:16:13.055012] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224178055010, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:16:13.106030] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58F6-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.110209] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E902-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:13.111860] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5277-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:13.112731] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E902-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:13.115511] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5277-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:13.121426] WDIAG [STORAGE] print_access_storage_log_ (ob_i_storage.cpp:907) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=10][errcode=0] access object storage cost too much time: del_file (ob_storage.cpp:839), uri=cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog/piece_d1001r1p37/checkpoint/checkpoint_info.1755224005638556002.obarc, size=0 byte, start_time=1755224173039108, cost_ts=82317 us, speed=0.00 MB/s, is_slow=1 [2025-08-15 10:16:13.122685] INFO [SHARE] get_backup_dest (ob_backup_connectivity.cpp:1268) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=15] success get backup dest(sql=SELECT authorization,extension FROM __all_backup_storage_info WHERE tenant_id = 1016 AND path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog' AND endpoint = 'host=cos.ap-beijing.myqcloud.com', tenant_id=1016, backup_dest={root_path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog", storage_info:{endpoint:"host=cos.ap-beijing.myqcloud.com", access_id:"access_id=AKIDiKl3A5ZXJ959EMuL3hJ1g9PTM5lFvjhT", extension:"appid=1254236265&checksum_type=md5", type:"COS", checksum_type:1, max_iops:0, max_bandwidth:0, role_arn:0x7f1bf395af90, external_id:0x7f1bf395b090}}) [2025-08-15 10:16:13.124094] INFO [SHARE] do_compare_and_swap_ (ob_inner_table_operator.cpp:937) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=19] compare and swap one column(key={key:{tenant_id:1016, dest_no:0}, incarnation:1, dest_id:1001, round_id:1, state:{status:"DOING"}, start_scn:{val:1752091205985713000, v:0}, checkpoint_scn:{val:1755224005638556002, v:0}, max_scn:{val:1755224059891515001, v:0}, compatible:{version:1}, base_piece_id:1, used_piece_id:37, piece_switch_interval:86400000000, frozen_input_bytes:44838045649, frozen_output_bytes:44838045649, active_input_bytes:331266514, active_output_bytes:331266514, deleted_input_bytes:0, deleted_output_bytes:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com", comment:""}, assignments="tenant_id = 1016, dest_no = 0, round_id = 1, incarnation = 1, dest_id = 1001, status = 'DOING', start_scn = 1752091205985713000, checkpoint_scn = 1755224058213484001, max_scn = 1755224133103752001, compatible = 1, base_piece_id = 1, used_piece_id = 37, piece_switch_interval = 86400000000, frozen_input_bytes = 44838045649, frozen_output_bytes = 44838045649, active_input_bytes = 332765182, active_output_bytes = 332765182, deleted_input_bytes = 0, deleted_output_bytes = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com', comment = ''", predicates="status = 'DOING' and round_id = 1 and checkpoint_scn = 1755224005638556002", affected_rows=1, sql=update __all_log_archive_progress set tenant_id = 1016, dest_no = 0, round_id = 1, incarnation = 1, dest_id = 1001, status = 'DOING', start_scn = 1752091205985713000, checkpoint_scn = 1755224058213484001, max_scn = 1755224133103752001, compatible = 1, base_piece_id = 1, used_piece_id = 37, piece_switch_interval = 86400000000, frozen_input_bytes = 44838045649, frozen_output_bytes = 44838045649, active_input_bytes = 332765182, active_output_bytes = 332765182, deleted_input_bytes = 0, deleted_output_bytes = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com', comment = '' where tenant_id = 1016 AND dest_no = 0 and status = 'DOING' and round_id = 1 and checkpoint_scn = 1755224005638556002) [2025-08-15 10:16:13.124254] WDIAG [STORAGE.TRANS] mvcc_sanity_check_ (ob_mvcc_row.cpp:937) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=25][errcode=-5024] find double insert node(ret=-5024, node=this=0x7f1d4d477e40 trans_version={val:4611686018427387903, v:0} scn={val:4611686018427387903, v:0} tx_id={txid:86702631} prev=(nil) next=(nil) modify_count=4294967295 acc_checksum=0 version=1755222868003467 type=0 flag=128 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:6} write_epoch=2, prev=this=0x7f1d4d477900 trans_version={val:1755224072901249001, v:0} scn={val:1755224072901249001, v:0} tx_id={txid:86701229} prev=0x7f1d4d477780 next=(nil) modify_count=43 acc_checksum=416833582 version=1755222868003467 type=0 flag=4 snapshot_barrier=1755224122949211000 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:1055} write_epoch=4, snapshot_version={val:1755224173123491000, v:0}, *this={this=0x7f1d4d466918 latch_=locked flag=11 first_dml=DELETE last_dml=INSERT update_since_compact=2 list_head=0x7f1d4d477900 latest_compact_node=0x7f1d4d4773a0 max_trans_version={val:1755224122949211001, v:0} max_trans_id=86701926 max_elr_trans_version={val:1755224122949211001, v:0} max_elr_trans_id=86701926 latest_compact_ts=1755224022720472 last_compact_cnt=41 total_trans_node_cnt=44 max_modify_scn={val:1755224072901249001, v:0} min_modify_scn={val:1755222868004408000, v:0}}) [2025-08-15 10:16:13.124293] WDIAG [STORAGE.TRANS] mvcc_write (ob_mvcc_row.cpp:999) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=32][errcode=-5024] mvcc write failed(ret=-5024, node=this=0x7f1d4d477e40 trans_version={val:4611686018427387903, v:0} scn={val:4611686018427387903, v:0} tx_id={txid:86702631} prev=(nil) next=(nil) modify_count=4294967295 acc_checksum=0 version=1755222868003467 type=0 flag=128 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:6} write_epoch=2, ctx={this:0x7f1deadc2dd0, ls_id:{id:1}, ls:0x7f1c55fba150, branch:0, timeout:1755224203024144, tablet_id:{id:350}, table_iter:0x7f1deadc1308, table_version:1750151411699448, mvcc_acc_ctx:{type:3, abs_lock_timeout_ts:1755224203024144, tx_lock_timeout_us:-1, snapshot:{version:{val:1755224173123491000, v:0}, tx_id:{txid:86702631}, scn:{branch:0, seq:4}}, tx_table_guards:{tx_table_guard:{tx_table:0x7f1c55fc5b10, epoch:1, mini_cache_:[0:{}, 1:{}, 2:{}, 3:{}]}, src_tx_table_guard:{tx_table:null, epoch:-1, mini_cache_:[0:{}, 1:{}, 2:{}, 3:{}]}, src_ls_handle:{ls_map_:null, ls_:null, mod_:-1}}, tx_id:{txid:86702631}, tx_desc:{this:0x7f1e7aaae920, tx_id:{txid:86702631}, state:2, addr:"10.203.1.34:2882", tenant_id:1015, session_id:3221544817, assoc_session_id:3221544817, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:{branch:0, seq:1}, op_sn:4, alloc_ts:1755224173123491, active_ts:1755224173123491, commit_ts:-1, finish_ts:-1, timeout_us:29999995, lock_timeout_us:-1, expire_ts:1755224203123486, coord_id:{id:-1}, parts:[{id:{id:1}, addr:"10.203.1.34:2882", epoch:509168309701642, first_scn:{branch:0, seq:3}, last_scn:{branch:0, seq:3}, last_touch_ts:1755224173123494}], exec_info_reap_ts:1755224173123493, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:1727264618, cluster_version:17180067073, seq_base:1755224173123490, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, conflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, modified_tables:[], last_rc_snapshot_version:{val:1755224173123491000, v:0}, ref:2}, tx_ctx:0x7f1c9defc7d0, mem_ctx:0x7f1c9defdd50, tx_scn:{branch:0, seq:6}, write_flag:{is_table_api:0, is_table_lock:0, is_mds:0, is_dml_batch_opt:0, is_insert_up:1, is_write_only_index:0, is_check_row_locked:0, is_lob_aux:0, is_skip_flush_redo:0, is_update_uk:0, is_update_pk_dop:0}, handle_start_time:-1, is_standby_read:false, lock_wait_start_ts:0, is_inited:true}, tablet_stat:{ls_id:0, tablet_id:0, query_cnt:0, merge_cnt:0, scan_logical_row_cnt:0, scan_physical_row_cnt:0, scan_micro_block_cnt:0, pushdown_micro_block_cnt:0, exist_row_total_table_cnt:0, exist_row_read_table_cnt:0, insert_row_cnt:0, update_row_cnt:0, delete_row_cnt:0}, is_read_store_ctx:false, update_full_column:false}) [2025-08-15 10:16:13.124364] WDIAG [STORAGE.TRANS] mvcc_write (ob_mvcc_engine.cpp:333) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=56][errcode=-5024] mvcc write failed(ret=-5024, arg={tx_id:{txid:86702631}, data:0x7f1deadbfcc0, old_row:null, modify_count:4294967295, acc_checksum:0, memstore_version:1755222868003467, seq_no:{branch:0, seq:6}, write_epoch:2, scn:{val:4611686018427387903, v:0}, column_cnt:23}) [2025-08-15 10:16:13.124379] WDIAG [STORAGE.TRANS] on_key_duplication_retry (ob_memtable_context.cpp:314) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=12][errcode=0] primary key duplication conflict(key={"BIGINT":1016},{"BIGINT":1001},{"BIGINT":1},{"BIGINT":37}, this={ObIMvccCtx={alloc_type=0 ctx_descriptor=0 min_table_version=1750151411699400 max_table_version=1750151411699448 trans_version={val:4611686018427387903, v:0} commit_version={val:0, v:0} lock_wait_start_ts=0 replay_compact_version={val:0, v:0}} end_code=0 tx_status=0 is_readonly=false ref=1 trans_id={txid:86702631} ls_id=1 row_callback[alloc:1, free:0, unsubmit:1] redo[fill:0,sync_succ:0, sync_fail:0] main_list_len=3 pending_log_size=289 callback_list:{cnt=1 stat:[tx_end=0, rollback_to=0, fast_commit=0, remove_memtable=0, ext_info_log=0] detail:[(log_epoch,length,logged,synced,appended,removed,unlog_removed,branch_removed)|0:(1,3,0,0,3,0,0,0)|]}}) [2025-08-15 10:16:13.124405] WDIAG [STORAGE] insert_row_without_rowkey_check (ob_tablet.cpp:4643) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=20][errcode=-5024] fail to set memtable(ret=-5024) [2025-08-15 10:16:13.124413] WDIAG [STORAGE] insert_row_to_tablet (ob_ls_tablet_service.cpp:5667) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=5][errcode=-5024] failed to write table row(ret=-5024, table id=350, col_descs=[cnt:23, column_id=18 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=19 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=20 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=21 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=16 {type:"TIMESTAMP", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=17 {type:"TIMESTAMP", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=22 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=23 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=24 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=25 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=26 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=27 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=28 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=29 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=30 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=31 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=32 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=33 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=34 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=35 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=36 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=37 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=38 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0], datum_row={row_flag:{flag:"INSERT", flag_type:0}, trans_id:{txid:0}, scan_index:0, mvcc_row_flag:{first:0, uncommitted:0, shadow:0, compact:0, ghost:0, last:0, reserved:0, flag:0}, snapshot_version:0, fast_filter_skipped:false, have_uncommited_row:false, group_idx:0, count:23, datum_buffer:{capacity:23, datums:0x7f18777cfea0, local_datums:0x7f18777cfaf0}, datums:[col_id=0:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf8a4, hex: F803000000000000, int: 1016}, col_id=1:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf8ac, hex: E903000000000000, int: 1001}, col_id=2:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf8b4, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=3:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf8bc, hex: 2500000000000000, int: 37}, col_id=4:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf8c4, hex: A31757F95D3C0600, int: 1755224173123491}, col_id=5:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf8cc, hex: A31757F95D3C0600, int: 1755224173123491}, col_id=6:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf8d4, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=7:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf8dc, hex: 0000000000000000, int: 0}, col_id=8:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf8e4, hex: 0000000000000000, int: 0}, col_id=9:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf8ec, hex: 683BF9A98FBA5B18, int: 1755201605985713000}, col_id=10:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf8f4, hex: E1C5083BFBCE5B18, int: 1755224058213484001}, col_id=11:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf8fc, hex: 4193D7AA0CCF5B18, int: 1755224133103752001}, col_id=12:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf904, hex: 683B483B24095C18, int: 1755288005985713000}, col_id=13:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf90c, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=14:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf914, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=15:{len: 4, flag: 0, null: 0, ptr: 0x7f18777cf91c, hex: 6E6F6E65, cstr: none}, col_id=16:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf920, hex: FE97D51300000000, int: 332765182}, col_id=17:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf928, hex: FE97D51300000000, int: 332765182}, col_id=18:{len: 6, flag: 0, null: 0, ptr: 0x7f18777cf930, hex: 414354495645, cstr: ACTIVE}, col_id=19:{len: 9, flag: 0, null: 0, ptr: 0x7f18777cf936, hex: 415641494C41424C45, cstr: AVAILABLE}, col_id=20:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf93f, hex: 0000000000000000, int: 0}, col_id=21:{len: 8, flag: 0, null: 0, ptr: 0x7f18777cf947, hex: 0000000000000000, int: 0}, col_id=22:{len: 135, flag: 0, null: 0, ptr: 0x7f18777cf94f, hex: 636F733A2F2F6261636B75702D6F6365616E626173652D313235343233363236352F6261636B75702D6F62636C75737465722F6F625F766563746F722F313732373236343631382F74656E616E745F696E6361726E6174696F6E5F312F313031362F636C6F673F686F73743D636F732E61702D6265696A696E672E6D7971636C, cstr: cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcl}]}) [2025-08-15 10:16:13.124495] WDIAG [STORAGE] insert_row (ob_ls_tablet_service.cpp:3241) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=55][errcode=-5024] failed to write row(ret=-5024) [2025-08-15 10:16:13.124522] INFO [STORAGE.TRANS] rollback (ob_memtable_context.cpp:851) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=6] memtable handle rollback to successfuly(from_seq_no={branch:0, seq:12}, to_seq_no={branch:0, seq:5}, remove_cnt=1, elapsed=0, this={ObIMvccCtx={alloc_type=0 ctx_descriptor=0 min_table_version=1750151411699400 max_table_version=1750151411699448 trans_version={val:4611686018427387903, v:0} commit_version={val:0, v:0} lock_wait_start_ts=0 replay_compact_version={val:0, v:0}} end_code=0 tx_status=0 is_readonly=false ref=0 trans_id={txid:86702631} ls_id=1 row_callback[alloc:1, free:0, unsubmit:1] redo[fill:0,sync_succ:0, sync_fail:0] main_list_len=2 pending_log_size=289 callback_list:{cnt=1 stat:[tx_end=0, rollback_to=1, fast_commit=0, remove_memtable=0, ext_info_log=0] detail:[(log_epoch,length,logged,synced,appended,removed,unlog_removed,branch_removed)|0:(1,2,0,0,3,1,1,0)|]}}) [2025-08-15 10:16:13.124720] INFO [SHARE] do_insert_or_update_row_ (ob_inner_table_operator.cpp:699) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2F-0-0] [lt=16] insert/update one row(row={key:{tenant_id:1016, dest_id:1001, round_id:1, piece_id:37}, incarnation:1, dest_no:0, file_count:0, start_scn:{val:1755201605985713000, v:0}, checkpoint_scn:{val:1755224058213484001, v:0}, max_scn:{val:1755224133103752001, v:0}, end_scn:{val:1755288005985713000, v:0}, compatible:{version:1}, input_bytes:332765182, output_bytes:332765182, status:{status:"ACTIVE"}, file_status:0, cp_file_id:0, cp_file_offset:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com"}, affected_rows=2, sql=INSERT INTO __all_log_archive_piece_files (tenant_id, dest_id, round_id, piece_id, incarnation, dest_no, file_count, start_scn, checkpoint_scn, max_scn, end_scn, compatible, input_bytes, output_bytes, status, file_status, cp_file_id, cp_file_offset, path) VALUES (1016, 1001, 1, 37, 1, 0, 0, 1755201605985713000, 1755224058213484001, 1755224133103752001, 1755288005985713000, 1, 332765182, 332765182, 'ACTIVE', 'AVAILABLE', 0, 0, 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com') ON DUPLICATE KEY UPDATE incarnation = 1, dest_no = 0, file_count = 0, start_scn = 1755201605985713000, checkpoint_scn = 1755224058213484001, max_scn = 1755224133103752001, end_scn = 1755288005985713000, compatible = 1, input_bytes = 332765182, output_bytes = 332765182, status = 'ACTIVE', file_status = 'AVAILABLE', cp_file_id = 0, cp_file_offset = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com') [2025-08-15 10:16:13.127164] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [438126][T1005_ArchiveSv][T1005][YB420ACB0122-000633820735554C-0-0] [lt=20] start do ObArchiveSchedulerService round(round=168833) [2025-08-15 10:16:13.128426] INFO [ARCHIVE] checkpoint_ (ob_tenant_archive_scheduler.cpp:682) [438126][T1005_ArchiveSv][T1005][YB420ACB0122-000633820735554C-0-0] [lt=14] schedule archive(round_info={key:{tenant_id:1006, dest_no:0}, incarnation:1, dest_id:1001, round_id:1, state:{status:"STOP"}, start_scn:{val:1748944774947721000, v:0}, checkpoint_scn:{val:1752037015224032000, v:0}, max_scn:{val:1752037085107831001, v:0}, compatible:{version:1}, base_piece_id:1, used_piece_id:36, piece_switch_interval:86400000000, frozen_input_bytes:16367268487, frozen_output_bytes:16367268487, active_input_bytes:0, active_output_bytes:0, deleted_input_bytes:0, deleted_output_bytes:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1006/clog?host=cos.ap-beijing.myqcloud.com", comment:""}) [2025-08-15 10:16:13.138301] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0D7-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.150629] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF078-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.187489] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D2F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.201974] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9E4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.204304] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86AD-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:13.204932] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5316-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:13.206790] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86AD-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:13.207430] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5316-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:13.272338] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FBC-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:13.276036] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FBC-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:13.311825] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C0-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:13.314302] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C0-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:13.443291] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C688E-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.462133] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:13.462146] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:13.462151] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:13.469113] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF87-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.469215] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2F-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.471650] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF87-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:13.471668] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF87-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.471752] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2F-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:13.471769] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A2F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.472365] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A0945-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}) [2025-08-15 10:16:13.472409] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A0945-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:16:13.503086] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0D5-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.503122] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E744-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.534116] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:13.534129] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:13.534135] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:13.534702] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:13.534717] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:13.534729] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:13.540244] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:13.540257] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:13.540263] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:13.546280] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:13.546291] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:13.546297] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:13.551636] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:13.551646] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:13.551651] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:13.594412] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7702-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224173373832002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224173373832003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224173373832003, v:0}}], last_dump_ts_=1755224168563190, force_dump=false) [2025-08-15 10:16:13.594441] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7702-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.597412] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7702-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224173373832002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224173373832003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224173373832003, v:0}}], last_dump_ts_=1755224168566379, force_dump=false) [2025-08-15 10:16:13.597432] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7702-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.599064] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7870-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.599086] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC069-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.599129] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C37-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.599187] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E1-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.599384] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7702-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.599491] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7702-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.602026] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7870-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.602119] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C37-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.602249] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E1-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.602299] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC069-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.603996] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7870-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.604072] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C37-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.604118] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7870-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.604164] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C37-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.604236] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E1-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.604429] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E1-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.604451] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC069-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.604550] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC069-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.611832] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:13.613091] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224173611756, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:16:13.613120] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224178613118, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:16:13.662973] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886A-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.665605] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886A-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:13.665624] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886A-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.665817] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A49-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.667393] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7660-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.668413] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A49-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:13.668429] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A49-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.670295] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7660-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.672156] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7660-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.672251] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7660-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:13.692174] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6891-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.703536] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDA3-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.708367] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:13.708382] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:13.708389] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:13.752500] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDB2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.752627] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF07C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.797639] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC64C-0-0] [lt=13] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.850246] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D32-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.939755] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C3-0-0] [lt=37] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.942417] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C3-0-0] [lt=70] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:13.942437] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C3-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.953015] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B38C-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:13.960267] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888F-0-0] [lt=41] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.962817] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888F-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:13.962833] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD888F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.992391] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB56-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.993646] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58FD-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.993992] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E1-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:13.994340] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9E0-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:13.994989] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB56-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:13.995007] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB56-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:13.996625] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E1-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:14.020615] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8391-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.023164] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8391-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224173874453001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224173874453001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224173874453001, v:0}}], last_dump_ts_=1755224168993130, force_dump=false) [2025-08-15 10:16:14.023192] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8391-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.025143] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8391-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.025246] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8391-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.106108] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58F7-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.118280] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E903-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:14.120863] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E903-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:14.120878] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5278-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:14.123428] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5278-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:14.138380] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0D8-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.141726] INFO [RS] gc_mview (ob_mview_maintenance_task.cpp:267) [438022][T1006_TimerWK1_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=15] mview maintenance task gc mview success(tenant_id_=1006, round_=105668, gc_mview_cost_us_=1329, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:16:14.150701] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF079-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.187547] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D30-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.202035] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9E5-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.212149] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86AE-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:14.212753] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5317-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:14.214664] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86AE-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:14.216396] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5317-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:14.281421] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FBD-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:14.283909] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FBD-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:14.283958] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95CC2-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}) [2025-08-15 10:16:14.284001] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95CC2-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:16:14.319632] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C1-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:14.323333] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C1-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:14.443366] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C688F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.462079] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224172873175001, replayable_scn:1755224172873175001, readable_scn:1755224172873175001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224174458530, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224172873175001, replayable_scn:1755224172873175001, readable_scn:1755224172873175001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224174458530, ora_rowscn_=1755224174021847000, content_changed=true) [2025-08-15 10:16:14.462255] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:14.462264] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:14.462272] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:14.463562] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:14.474416] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF88-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:14.474445] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A30-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:14.476997] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF88-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:14.477004] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A30-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:14.477013] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF88-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:14.477017] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A30-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:14.503164] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0D6-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.503200] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E745-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.511525] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF226-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.534022] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224172553512001, replayable_scn:1755224172553512001, readable_scn:1755224172553512001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224174531117, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224172553512001, replayable_scn:1755224172553512001, readable_scn:1755224172553512001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224174531117, ora_rowscn_=1755224173669247000, content_changed=true) [2025-08-15 10:16:14.534225] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:14.534242] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:14.534250] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:14.534600] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224172372569002, replayable_scn:1755224172372569002, readable_scn:1755224172372569002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224174531117, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224172372569002, replayable_scn:1755224172372569002, readable_scn:1755224172372569002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224174531117, ora_rowscn_=1755224173600874000, content_changed=true) [2025-08-15 10:16:14.534755] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:14.534768] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:14.534777] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:14.535623] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:14.540083] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A97-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80273438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}) [2025-08-15 10:16:14.540130] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A97-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=50, ddl_arg=NULL) [2025-08-15 10:16:14.540234] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:14.540248] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:14.540254] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:14.546294] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224172372569002, replayable_scn:1755224172372569002, readable_scn:1755224172372569002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224174542703, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224172372569002, replayable_scn:1755224172372569002, readable_scn:1755224172372569002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224174542703, ora_rowscn_=1755224173600874000, content_changed=true) [2025-08-15 10:16:14.546447] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:14.546458] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:14.546465] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:14.551531] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224172372569002, replayable_scn:1755224172372569002, readable_scn:1755224172372569002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224174547969, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224172372569002, replayable_scn:1755224172372569002, readable_scn:1755224172372569002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224174547969, ora_rowscn_=1755224173600874000, content_changed=true) [2025-08-15 10:16:14.551697] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:14.551706] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:14.551713] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:14.553006] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:14.600478] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7703-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.602279] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:104) [17458][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=11] start checking lost replicas [2025-08-15 10:16:14.603497] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7703-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.605102] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7871-0-0] [lt=9] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224174375477002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224174375477003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224174375477003, v:0}}], last_dump_ts_=1755224169574521, force_dump=false) [2025-08-15 10:16:14.605108] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C38-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224174375477001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224174375477002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224174375477002, v:0}}], last_dump_ts_=1755224169574438, force_dump=false) [2025-08-15 10:16:14.605127] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7871-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.605133] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C38-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.605394] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E2-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.605548] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.605620] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7703-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.605698] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7703-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.608124] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7871-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.608199] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C38-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224174375477001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224174375477002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224174375477002, v:0}}], last_dump_ts_=1755224169577486, force_dump=false) [2025-08-15 10:16:14.608221] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C38-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.608321] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E2-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.608487] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06A-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.610078] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7871-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.610193] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C38-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.610222] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7871-0-0] [lt=24] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.610266] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E2-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.610383] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E2-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.610392] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C38-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.610534] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.610613] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.632422] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:14.633648] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224174632343, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:16:14.633671] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224179633669, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:16:14.668313] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886B-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:14.670840] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886B-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:14.670856] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886B-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:14.671669] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4A-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:14.673268] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7661-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224174444851002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224174444851003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224174444851003, v:0}}], last_dump_ts_=1755224169643407, force_dump=false) [2025-08-15 10:16:14.673292] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7661-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.674242] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4A-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:14.674265] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4A-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:14.676279] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7661-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224174444851001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224174444851002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224174444851002, v:0}}], last_dump_ts_=1755224169646338, force_dump=false) [2025-08-15 10:16:14.676301] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7661-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.678191] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7661-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.678302] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7661-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:14.692250] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6892-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.703576] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDA4-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.708265] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224173373832003, replayable_scn:1755224173373832003, readable_scn:1755224173373832003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224174704752, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224173373832003, replayable_scn:1755224173373832003, readable_scn:1755224173373832003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224174704752, ora_rowscn_=1755224174606875000, content_changed=true) [2025-08-15 10:16:14.708444] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:14.708457] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:14.708464] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:14.709777] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:14.728868] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476036][T1007_DRService][T1007][YB420ACB0122-0006338215870D0F-0-0] [lt=15] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1007, thread_idx=1) [2025-08-15 10:16:14.732525] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [476036][T1007_DRService][T1007][YB420ACB0122-0006338215870D10-0-0] [lt=12] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2411, tenant_id=1007, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:14.734843] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [476036][T1007_DRService][T1007][YB420ACB0122-0006338215870D10-0-0] [lt=14] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2298, tenant_id=1008, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:14.752573] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDB3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.752698] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF07D-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.850323] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D33-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.945115] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C4-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:14.947651] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C4-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:14.947670] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C4-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:14.954418] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B38D-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:14.965510] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8890-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:14.968044] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8890-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:14.968063] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8890-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:14.993722] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58FE-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.994414] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9E1-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:14.997800] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB57-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.000362] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB57-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:15.000379] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB57-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.001983] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E2-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:15.004458] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E2-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:15.009387] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=4] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:15.026213] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8392-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224174876176001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224174876176001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224174876176001, v:0}}], last_dump_ts_=1755224169996317, force_dump=false) [2025-08-15 10:16:15.026237] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8392-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.028632] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8392-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.030503] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8392-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.030601] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8392-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.099641] INFO [RS.LB] idle (ob_root_balancer.cpp:87) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=31] root balance idle(idle_time=10000000) [2025-08-15 10:16:15.099656] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:70) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=10] start check unit migrate finish [2025-08-15 10:16:15.099665] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=5] try check migrate unit finish by tenant(tenant_id=1) [2025-08-15 10:16:15.106184] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58F8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.109447] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=6] try check migrate unit finish by tenant(tenant_id=1004) [2025-08-15 10:16:15.126179] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E904-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:15.128672] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E904-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:15.131149] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5279-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:15.134809] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5279-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:15.138447] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0D9-0-0] [lt=8] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.138620] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1006) [2025-08-15 10:16:15.150776] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF07A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.163817] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1008) [2025-08-15 10:16:15.187623] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D31-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.189291] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1012) [2025-08-15 10:16:15.202094] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9E6-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.214639] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1014) [2025-08-15 10:16:15.220028] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86AF-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:15.221733] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5318-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:15.222535] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86AF-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:15.224269] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5318-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:15.240278] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1016) [2025-08-15 10:16:15.248012] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C59-0-0] [lt=13] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1011, thread_idx=0) [2025-08-15 10:16:15.249311] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1011, only_for_display=false) [2025-08-15 10:16:15.250567] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:15.265780] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1018) [2025-08-15 10:16:15.266571] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.267728] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.268871] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.268894] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1011, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:15.268909] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.268919] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1011, ls_id:{id:1}, replicas:[{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:15.269033] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=88] build ls locality stat map(tenant_id=1011, ls_id={id:1}) [2025-08-15 10:16:15.269045] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.269057] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.269066] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.269075] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:15.269153] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.269166] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.269174] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.269179] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.269188] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3773, tenant_id=1011, ls_id={id:1}) [2025-08-15 10:16:15.269196] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:15.270664] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=21353, tenant_id=1011, acc_dr_task=0) [2025-08-15 10:16:15.271919] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1012, only_for_display=false) [2025-08-15 10:16:15.273158] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:15.288420] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.289288] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FBE-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:15.289571] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.290710] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.290731] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:15.290747] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.290758] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1}, replicas:[{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:15.290873] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=90] build ls locality stat map(tenant_id=1012, ls_id={id:1}) [2025-08-15 10:16:15.290882] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.290895] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.290904] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.290914] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:15.290999] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.291013] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.291022] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.291030] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.291038] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3772, tenant_id=1012, ls_id={id:1}) [2025-08-15 10:16:15.291047] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:15.291428] INFO [RS] check (ob_alter_locality_finish_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81556-0-0] [lt=14] start to check alter locality finish [2025-08-15 10:16:15.291810] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FBE-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:15.292582] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81557-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.292606] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:15.293728] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81557-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.296038] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81559-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.297180] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81559-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.299479] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8155B-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.300841] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8155B-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.303613] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8155D-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.304754] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8155D-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.307054] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8155F-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.308192] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8155F-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.310453] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81561-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.311584] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81561-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.313841] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81563-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.314720] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.314988] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81563-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.315877] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.317022] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.317043] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:15.317060] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.317070] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1001}, replicas:[{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:15.317179] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=85] build ls locality stat map(tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:16:15.317191] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.317204] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.317213] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.317268] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81565-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.317223] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:15.317307] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.317323] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.317331] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.317339] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.317347] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3782, tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:16:15.317356] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:15.318407] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81565-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.318787] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:15.319563] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81565-0-0] [lt=13] start check shrink resource pool [2025-08-15 10:16:15.319580] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81566-0-0] [lt=10] start check shrink resource pool(tenant_id=1) [2025-08-15 10:16:15.320681] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81566-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.321813] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81566-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.321863] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81567-0-0] [lt=12] start check shrink resource pool(tenant_id=1004) [2025-08-15 10:16:15.322987] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81567-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.324134] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81567-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.324184] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81568-0-0] [lt=12] start check shrink resource pool(tenant_id=1006) [2025-08-15 10:16:15.325284] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81568-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.326429] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81568-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.326480] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81569-0-0] [lt=12] start check shrink resource pool(tenant_id=1008) [2025-08-15 10:16:15.327590] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81569-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.328674] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C2-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:15.328718] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81569-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.328767] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156A-0-0] [lt=12] start check shrink resource pool(tenant_id=1012) [2025-08-15 10:16:15.329862] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156A-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.331006] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156A-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.331068] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156B-0-0] [lt=12] start check shrink resource pool(tenant_id=1014) [2025-08-15 10:16:15.331196] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C2-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:15.332200] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156B-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.333335] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156B-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.333385] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156C-0-0] [lt=13] start check shrink resource pool(tenant_id=1016) [2025-08-15 10:16:15.334489] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156C-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.335618] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156C-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.335667] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156D-0-0] [lt=13] start check shrink resource pool(tenant_id=1018) [2025-08-15 10:16:15.336805] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156D-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.337968] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156D-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.338018] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:99) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156D-0-0] [lt=11] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.338031] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156D-0-0] [lt=10] start check primary zone [2025-08-15 10:16:15.338045] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156E-0-0] [lt=8] start check primary zone(tenant_id=1) [2025-08-15 10:16:15.339152] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156E-0-0] [lt=6][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.339205] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156F-0-0] [lt=12] start check primary zone(tenant_id=1004) [2025-08-15 10:16:15.340326] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8156F-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.340374] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81570-0-0] [lt=12] start check primary zone(tenant_id=1006) [2025-08-15 10:16:15.341147] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.341476] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81570-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.341524] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81571-0-0] [lt=12] start check primary zone(tenant_id=1008) [2025-08-15 10:16:15.342316] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.342620] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81571-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.342667] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81572-0-0] [lt=12] start check primary zone(tenant_id=1012) [2025-08-15 10:16:15.343483] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.343506] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:15.343520] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.343530] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1002}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:15.343639] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=85] build ls locality stat map(tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:16:15.343651] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.343662] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.343671] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.343680] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:15.343758] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.343772] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.343780] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.343776] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81572-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.343788] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.343797] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3805, tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:16:15.343806] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:15.343823] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81573-0-0] [lt=11] start check primary zone(tenant_id=1014) [2025-08-15 10:16:15.344924] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81573-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.344977] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81574-0-0] [lt=15] start check primary zone(tenant_id=1016) [2025-08-15 10:16:15.345397] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:15.346095] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81574-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.346144] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=12] start check primary zone(tenant_id=1018) [2025-08-15 10:16:15.347250] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:15.347303] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:166) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=12] start do unit balance [2025-08-15 10:16:15.353557] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:350) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=10] start distribute for server status change [2025-08-15 10:16:15.353658] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:401) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=12] finish distribute for server status change(ret=0) [2025-08-15 10:16:15.353669] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:200) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=8] finish do unit balance(ret=0) [2025-08-15 10:16:15.360366] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:115) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=3] start to do tenant group unit balance [2025-08-15 10:16:15.360412] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=14] start to do tenant group unit balance(zone=zone1) [2025-08-15 10:16:15.360428] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=10] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.203.1.34:2882", disk_in_use_:3126853632, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:15.360449] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=13] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:15.360481] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=12] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:15.360499] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=9] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:15.360510] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=8] start do non-tenantgroup unit balance task(zone=zone1) [2025-08-15 10:16:15.360521] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=8] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:15.360542] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=13] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.1.34:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:15.360585] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=33] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.203.1.34:2882", disk_in_use_:3126853632, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:15.360593] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=6] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.1.34:2882", disk_in_use_:3126853632, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:15.360607] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=9] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:15.360621] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=10] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1006, unit_config:{unit_config_id:1005, name:"config_opsone_zone1_s_2_4_ywx", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7988, unit_config:0x7f1e9aa71c38, pool:0x7f1c3beb2f18}, {unit_id:1008, tenant_id:1006, resource_pool_id:1008, unit_config:{unit_config_id:1008, name:"config_test_zone1_s_2_4_emp", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f77d8, unit_config:0x7f1e9aa71998, pool:0x7f1c3beb27a8}, {unit_id:1011, tenant_id:1008, resource_pool_id:1011, unit_config:{unit_config_id:1011, name:"config_mcp_zone1_s_4_8_tmc", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7550, unit_config:0x7f1e9aa716f8, pool:0x7f1e443fda40}, {unit_id:1016, tenant_id:1012, resource_pool_id:1016, unit_config:{unit_config_id:1017, name:"config_oms_zone1_s_4_8_gzp", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f73a0, unit_config:0x7f1e9aa71458, pool:0x7f1e443fd2d0}, {unit_id:1019, tenant_id:1014, resource_pool_id:1019, unit_config:{unit_config_id:1019, name:"config_grafana_test_zone1_s_1_2_pri", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7118, unit_config:0x7f1e9aa71298, pool:0x7f1e443fc7a8}, {unit_id:1024, tenant_id:1016, resource_pool_id:1024, unit_config:{unit_config_id:1023, name:"config_temporalio_zone1_s_2_4_ykw", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6ce0, unit_config:0x7f1e9aa70f18, pool:0x7f1e74dff2d0}, {unit_id:1026, tenant_id:1018, resource_pool_id:1026, unit_config:{unit_config_id:1027, name:"config_media_zone1_s_2_4_kmn", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6b30, unit_config:0x7f1e9aa70b98, pool:0x7f1e74dfeb60}], not_grant_units=[], available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:15.360679] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=34] start to do tenant group unit balance(zone=zone2) [2025-08-15 10:16:15.360692] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=9] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.203.4.38:2882", disk_in_use_:3114270720, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:15.360705] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:15.360722] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=9] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:15.360734] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=7] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:15.360743] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone2) [2025-08-15 10:16:15.360748] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:15.360761] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.38:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:15.360793] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=26] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.203.4.38:2882", disk_in_use_:3114270720, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:15.360799] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=4] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.38:2882", disk_in_use_:3114270720, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:15.360809] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=7] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:15.360820] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=8] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1004, unit_config:{unit_config_id:1004, name:"config_opsone_zone2_s_2_4_wag", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7b38, unit_config:0x7f1e9aa71d18, pool:0x7f1c3beb3688}, {unit_id:1009, tenant_id:1006, resource_pool_id:1009, unit_config:{unit_config_id:1009, name:"config_test_zone2_s_2_4_yey", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7700, unit_config:0x7f1e9aa718b8, pool:0x7f1c3beb23f0}, {unit_id:1010, tenant_id:1008, resource_pool_id:1010, unit_config:{unit_config_id:1010, name:"config_mcp_zone2_s_4_8_bjb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7628, unit_config:0x7f1e9aa717d8, pool:0x7f1c3beb2038}, {unit_id:1018, tenant_id:1012, resource_pool_id:1018, unit_config:{unit_config_id:1016, name:"config_oms_zone2_s_4_8_kbx", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f71f0, unit_config:0x7f1e9aa71538, pool:0x7f1e443fcb60}, {unit_id:1021, tenant_id:1014, resource_pool_id:1021, unit_config:{unit_config_id:1021, name:"config_grafana_test_zone2_s_1_2_yyc", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f6f68, unit_config:0x7f1e9aa710d8, pool:0x7f1e443fc038}, {unit_id:1023, tenant_id:1016, resource_pool_id:1023, unit_config:{unit_config_id:1022, name:"config_temporalio_zone2_s_2_4_lsv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6db8, unit_config:0x7f1e9aa70ff8, pool:0x7f1e74dff688}, {unit_id:1027, tenant_id:1018, resource_pool_id:1027, unit_config:{unit_config_id:1025, name:"config_media_zone2_s_2_4_yzd", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6a58, unit_config:0x7f1e9aa70d58, pool:0x7f1e74dfe7a8}], not_grant_units=[], available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:15.360871] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=30] start to do tenant group unit balance(zone=zone3) [2025-08-15 10:16:15.360880] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.203.4.39:2882", disk_in_use_:3013607424, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:15.360891] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:15.360907] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=9] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:15.360919] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=7] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:15.360928] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone3) [2025-08-15 10:16:15.360932] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=2] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:15.360951] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=14] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.39:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:15.360983] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=25] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.203.4.39:2882", disk_in_use_:3013607424, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:15.360996] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=11] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.39:2882", disk_in_use_:3013607424, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:15.361007] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:15.361016] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=6] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1005, unit_config:{unit_config_id:1006, name:"config_opsone_zone3_s_2_4_vxv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7a60, unit_config:0x7f1e9aa71b58, pool:0x7f1c3beb32d0}, {unit_id:1007, tenant_id:1006, resource_pool_id:1007, unit_config:{unit_config_id:1007, name:"config_test_zone3_s_2_4_mui", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f78b0, unit_config:0x7f1e9aa71a78, pool:0x7f1c3beb2b60}, {unit_id:1012, tenant_id:1008, resource_pool_id:1012, unit_config:{unit_config_id:1012, name:"config_mcp_zone3_s_4_8_seu", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7478, unit_config:0x7f1e9aa71618, pool:0x7f1e443fd688}, {unit_id:1017, tenant_id:1012, resource_pool_id:1017, unit_config:{unit_config_id:1018, name:"config_oms_zone3_s_4_8_nvb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f72c8, unit_config:0x7f1e9aa71378, pool:0x7f1e443fcf18}, {unit_id:1020, tenant_id:1014, resource_pool_id:1020, unit_config:{unit_config_id:1020, name:"config_grafana_test_zone3_s_1_2_xbz", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7040, unit_config:0x7f1e9aa711b8, pool:0x7f1e443fc3f0}, {unit_id:1022, tenant_id:1016, resource_pool_id:1022, unit_config:{unit_config_id:1024, name:"config_temporalio_zone3_s_2_4_rxz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6e90, unit_config:0x7f1e9aa70e38, pool:0x7f1e74dffa40}, {unit_id:1025, tenant_id:1018, resource_pool_id:1025, unit_config:{unit_config_id:1026, name:"config_media_zone3_s_2_4_ukz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6c08, unit_config:0x7f1e9aa70c78, pool:0x7f1e74dfef18}], not_grant_units=[], available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:15.367658] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.368807] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.369967] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.369988] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:15.370004] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.370015] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1003}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:15.370123] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=84] build ls locality stat map(tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:16:15.370134] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.370147] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.370157] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.370166] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:15.370245] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.370258] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.370266] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.370275] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.370282] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3765, tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:16:15.370291] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:15.371823] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=101145, tenant_id=1012, acc_dr_task=0) [2025-08-15 10:16:15.436979] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227775435791}) [2025-08-15 10:16:15.436996] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:15.437008] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:15.443030] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227775441846}) [2025-08-15 10:16:15.443046] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:15.443058] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:15.443444] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6890-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.444237] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227775443070}) [2025-08-15 10:16:15.444250] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:15.444261] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:15.462312] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:15.462328] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:15.462333] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:15.472897] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A08DF-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}) [2025-08-15 10:16:15.472959] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A08DF-0-0] [lt=39] [DDL] execute ddl like stmt(ret=0, cost=65, ddl_arg=NULL) [2025-08-15 10:16:15.479916] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF89-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.480822] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A31-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.482469] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF89-0-0] [lt=68] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:15.482486] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF89-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.484447] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A31-0-0] [lt=89] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:15.484467] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A31-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.503234] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0D7-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.503275] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E746-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.534251] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:15.534272] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:15.534279] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:15.534870] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:15.534881] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:15.534889] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:15.540628] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224173373832003, replayable_scn:1755224173373832003, readable_scn:1755224173373832003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224175537597, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224173373832003, replayable_scn:1755224173373832003, readable_scn:1755224173373832003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224175537597, ora_rowscn_=1755224174602654000, content_changed=true) [2025-08-15 10:16:15.540804] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:15.540821] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:15.540829] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:15.540841] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1018, cost_time_us=188, return_code_array=[0, 0, 0], broadcast_times=2) [2025-08-15 10:16:15.542207] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:15.546400] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:15.546411] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:15.546416] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:15.551726] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:15.551737] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:15.551742] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:15.559650] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED7-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1007, thread_idx=0) [2025-08-15 10:16:15.560950] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1007, only_for_display=false) [2025-08-15 10:16:15.562180] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:15.576619] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.577762] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.578905] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.578927] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1007, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:15.578948] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=13] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.578957] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1007, ls_id:{id:1}, replicas:[{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:15.579058] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=78] build ls locality stat map(tenant_id=1007, ls_id={id:1}) [2025-08-15 10:16:15.579065] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.579077] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.579085] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.579094] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:15.579171] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.579184] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.579191] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.579198] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.579205] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3734, tenant_id=1007, ls_id={id:1}) [2025-08-15 10:16:15.579214] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:15.580287] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19336, tenant_id=1007, acc_dr_task=0) [2025-08-15 10:16:15.581502] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1008, only_for_display=false) [2025-08-15 10:16:15.582693] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:15.596433] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.597588] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.598738] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.598764] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:15.598776] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.598788] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1}, replicas:[{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:15.598893] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=83] build ls locality stat map(tenant_id=1008, ls_id={id:1}) [2025-08-15 10:16:15.598901] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.598914] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.598924] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.598936] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:15.599019] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.599034] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.599045] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.599052] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.599061] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3787, tenant_id=1008, ls_id={id:1}) [2025-08-15 10:16:15.599071] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:15.600178] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:15.606722] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7704-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.609875] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7704-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.611305] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7872-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.611352] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C39-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.611352] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E3-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.611546] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.612054] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7704-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.612168] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7704-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.614316] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E3-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.614357] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C39-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.614392] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7872-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.614584] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06B-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.616278] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E3-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.616303] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C39-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.616375] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E3-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.616401] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C39-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.616490] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7872-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.616538] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.616577] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7872-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.616610] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.621097] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.622238] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.623394] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.623417] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:15.623430] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.623443] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1001}, replicas:[{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:15.623547] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=82] build ls locality stat map(tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:16:15.623555] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.623566] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.623577] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.623583] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:15.623661] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.623675] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.623685] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.623690] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.623694] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3727, tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:16:15.623700] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:15.624807] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:15.627457] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:15.628702] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224175627379, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:16:15.628729] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224180628728, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:16:15.645474] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.646622] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.647770] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.647795] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:15.647813] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.647825] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1002}, replicas:[{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:15.647929] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=82] build ls locality stat map(tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:16:15.647943] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.647957] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=12] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.647964] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.647975] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:15.648053] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.648066] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.648075] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.648081] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.648090] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3752, tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:16:15.648097] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:15.649177] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:15.669803] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.670956] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.672101] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:15.672124] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:15.672141] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.672153] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1003}, replicas:[{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:15.672256] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=81] build ls locality stat map(tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:16:15.672269] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.672283] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.672294] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:15.672306] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:15.672384] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.672399] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.672409] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.672420] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=9] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:15.672425] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3754, tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:16:15.672436] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:15.673585] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93282, tenant_id=1008, acc_dr_task=0) [2025-08-15 10:16:15.673564] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886C-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.676192] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886C-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:15.676209] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886C-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.677098] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4B-0-0] [lt=47] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.679321] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7662-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.679700] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4B-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:15.679715] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4B-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.682307] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7662-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.684433] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7662-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.684533] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7662-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:15.692326] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6893-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.703654] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDA5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.708522] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:15.708535] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:15.708541] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:15.722132] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438022][T1006_TimerWK1_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=7] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:15.752648] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDB4-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.752771] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF07E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.802587] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC64D-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.850396] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D34-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.932004] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224175548103, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224173373832002, replayable_scn:1755224173373832002, readable_scn:1755224173373832002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:15.950431] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C5-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.952084] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:15.952936] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C5-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:15.952961] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C5-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.953339] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224175952008, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:16:15.953363] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224180953361, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:16:15.955855] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B38E-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:15.970709] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8891-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.973271] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8891-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:15.973287] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8891-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:15.993795] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58FF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:15.994492] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9E2-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.003308] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB58-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.005559] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2845) [17349][TimerWK2_HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=5] need remove task(remove_task_ids=[]) [2025-08-15 10:16:16.005575] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2864) [17349][TimerWK2_HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=11] remove all timeout ddl task succeed [2025-08-15 10:16:16.006053] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB58-0-0] [lt=70] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:16.006073] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB58-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.010805] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E3-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:16.013402] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E3-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:16.031678] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8393-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.034542] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8393-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.036918] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8393-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.037095] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8393-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.057497] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:16.058799] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224176048797, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:16:16.058824] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224181058822, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:16:16.058840] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224181058838, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:16.106262] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58F9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.118128] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [2619609][T1011_BackupDat][T1011][YB420ACB0122-0006338CC14639D2-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:16.134044] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E905-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:16.136547] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E905-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:16.138522] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0DA-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.140078] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD527A-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:16.143321] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD527A-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:16.150851] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF07B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.187698] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D32-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.194915] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:16.196162] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224176194835, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:16:16.196195] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224181196193, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:16:16.197174] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:16.198350] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224176197101, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:16:16.198371] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224181198370, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:16.202169] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9E7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.228463] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B0-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:16.229678] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5319-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:16.231005] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B0-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:16.232230] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5319-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:16.266876] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [1487706][T1003_BackupDat][T1003][YB420ACB0122-0006337FE347F6D8-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:16.284483] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA28-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}) [2025-08-15 10:16:16.284525] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA28-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=47, ddl_arg=NULL) [2025-08-15 10:16:16.297091] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FBF-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:16.299592] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FBF-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:16.336588] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C3-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:16.339087] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C3-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:16.372381] INFO [RS] gc_mview (ob_mview_maintenance_task.cpp:267) [727957][T1018_TimerWK1_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=12] mview maintenance task gc mview success(tenant_id_=1018, round_=63408, gc_mview_cost_us_=1231, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:16:16.443520] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6891-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.462315] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:16.462327] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:16.462332] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:16.485220] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8A-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.487142] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A32-0-0] [lt=44] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.487777] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8A-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:16.487797] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8A-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.489658] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A32-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:16.489676] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A32-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.503310] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0D8-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.503350] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E747-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.511808] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF227-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.534314] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:16.534332] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:16.534338] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:16.534883] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:16.534896] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:16.534905] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:16.540340] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:16.540353] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:16.540359] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:16.540564] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A9A-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81054688GB}) [2025-08-15 10:16:16.540605] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A9A-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2025-08-15 10:16:16.546463] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:16.546474] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:16.546483] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:16.551768] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:16.551778] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:16.551787] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:16.613157] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7705-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.616200] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7705-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.617373] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E4-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224176378593002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224176378593002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224176378593002, v:0}}], last_dump_ts_=1755224171587214, force_dump=false) [2025-08-15 10:16:16.617395] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E4-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.617394] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.617469] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7873-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.617545] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.618221] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7705-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.618312] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7705-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.620440] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7873-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.620500] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E4-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224176378593002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224176378593002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224176378593002, v:0}}], last_dump_ts_=1755224171590193, force_dump=false) [2025-08-15 10:16:16.620521] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E4-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.620571] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06C-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224176378593001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224176378593002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224176378593002, v:0}}], last_dump_ts_=1755224171589737, force_dump=false) [2025-08-15 10:16:16.620592] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06C-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.620913] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.622388] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7873-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.622478] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7873-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.622484] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E4-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.622568] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E4-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.622588] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.622666] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.622863] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.622948] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.623896] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [728059][T1017_BackupDat][T1017][YB420ACB0122-000633A355859846-0-0] [lt=12] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:16.643733] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [728058][T1017_BACKUP_SC][T1017][YB420ACB0122-000633A35572C1CC-0-0] [lt=17] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:16:16.679052] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886D-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.681670] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886D-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:16.681685] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886D-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.682430] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4C-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.685020] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4C-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:16.685039] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4C-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.685529] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7663-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.688622] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7663-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.690558] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7663-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.690667] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7663-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:16.692402] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6894-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.703731] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDA6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.708628] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:16.708642] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:16.708648] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:16.752721] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDB5-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.752846] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF07F-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.786386] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709535][T1014_TimerWK2_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=10] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:16.829100] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:16.836309] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:16.837534] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224176836234, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:16:16.837558] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224181837556, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:16:16.850471] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D35-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.955727] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C6-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.957273] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B38F-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:16.958250] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C6-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:16.958268] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C6-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.976339] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8892-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.979172] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8892-0-0] [lt=92] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:16.979191] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8892-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:16.993870] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5900-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:16.994568] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9E3-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.008815] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB59-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.011334] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB59-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:17.011352] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB59-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.018757] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E4-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:17.021269] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E4-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:17.038118] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8394-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.041127] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8394-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.043049] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8394-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.043129] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8394-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.106338] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58FA-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.138592] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0DB-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.141906] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E906-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:17.144441] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E906-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:17.148577] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD527B-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:17.150923] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF07C-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.151046] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD527B-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:17.164378] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727135][T1016_TimerWK3_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:17.187777] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D33-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.202241] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9E8-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.212062] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727957][T1018_TimerWK1_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:17.236496] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B1-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:17.237623] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD531A-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:17.237668] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B1-0-0] [lt=13] log restore source is empty, just skip(ret=-4018, user_tenant_id=1006) [2025-08-15 10:16:17.240143] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD531A-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:17.240147] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B1-0-0] [lt=9] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:17.266339] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619488][T1012_TimerWK3_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=30] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:17.304998] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC0-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:17.307498] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC0-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:17.326612] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487605][T1004_TimerWK2_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=30] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:17.327592] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [1487604][T1004_TimerWK1_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=14] mview_mds_op(tx_count=0) [2025-08-15 10:16:17.327606] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [1487604][T1004_TimerWK1_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=10] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=866) [2025-08-15 10:16:17.341201] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:17.342446] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224177341125, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:16:17.342471] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224182342468, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:16:17.344473] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C4-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:17.347018] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C4-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:17.432799] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:17.434051] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224177432724, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:16:17.434077] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224182434075, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:16:17.442180] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E63-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224177441574000, start_id=1755224177441574000, end_id=1755224177441574001) [2025-08-15 10:16:17.442861] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E63-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224177441574000, v:0}, tenant_id=1017) [2025-08-15 10:16:17.443595] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6892-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.462160] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224175877105003, replayable_scn:1755224175877105003, readable_scn:1755224175877105003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224177459479, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224175877105003, replayable_scn:1755224175877105003, readable_scn:1755224175877105003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224177459479, ora_rowscn_=1755224177040117000, content_changed=true) [2025-08-15 10:16:17.462335] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:17.462345] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:17.462355] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:17.473444] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A08E2-0-0] [lt=14] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}) [2025-08-15 10:16:17.473483] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A08E2-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:16:17.490575] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8B-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.492386] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A33-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.493136] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8B-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:17.493155] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8B-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.495044] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A33-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:17.495059] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A33-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.503385] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0D9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.503423] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E748-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.534187] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224175557588001, replayable_scn:1755224175557588001, readable_scn:1755224175557588001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224177531014, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224175557588001, replayable_scn:1755224175557588001, readable_scn:1755224175557588001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224177531014, ora_rowscn_=1755224176686874000, content_changed=true) [2025-08-15 10:16:17.534372] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:17.534384] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:17.534397] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:17.534844] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224175376934002, replayable_scn:1755224175376934002, readable_scn:1755224175376934002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224177532067, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224175376934002, replayable_scn:1755224175376934002, readable_scn:1755224175376934002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224177532067, ora_rowscn_=1755224176619583000, content_changed=true) [2025-08-15 10:16:17.535017] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:17.535028] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:17.535036] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:17.536321] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:17.540371] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:17.540384] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:17.540389] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:17.546407] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224175376934002, replayable_scn:1755224175376934002, readable_scn:1755224175376934002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224177543546, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224175376934002, replayable_scn:1755224175376934002, readable_scn:1755224175376934002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224177543546, ora_rowscn_=1755224176619583000, content_changed=true) [2025-08-15 10:16:17.546644] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:17.546659] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:17.546665] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:17.547975] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:17.551668] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224175376934002, replayable_scn:1755224175376934002, readable_scn:1755224175376934002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224177548813, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224175376934002, replayable_scn:1755224175376934002, readable_scn:1755224175376934002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224177548813, ora_rowscn_=1755224176619583000, content_changed=true) [2025-08-15 10:16:17.551843] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:17.551855] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:17.551860] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:17.576740] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E6C-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224177576131000, start_id=1755224177576131000, end_id=1755224177576131001) [2025-08-15 10:16:17.577444] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E6C-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224177576131000, v:0}, tenant_id=1015) [2025-08-15 10:16:17.577861] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66E-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224177577184000, start_id=1755224177577184000, end_id=1755224177577184001) [2025-08-15 10:16:17.578528] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66E-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224177577184000, v:0}, tenant_id=1005) [2025-08-15 10:16:17.619348] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7706-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.622359] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7706-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.623486] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E5-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.623613] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06D-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224177379523002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224177379523002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224177379523002, v:0}}], last_dump_ts_=1755224172592859, force_dump=false) [2025-08-15 10:16:17.623636] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06D-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.623772] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7874-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.623882] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.624352] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7706-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.624448] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7706-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.626658] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E5-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.626666] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.626886] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.626909] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7874-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224177379523004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224177379523004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224177379523003, v:0}}], last_dump_ts_=1755224172595865, force_dump=false) [2025-08-15 10:16:17.626934] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7874-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.628608] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E5-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.628661] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.628702] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E5-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.628745] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.628892] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.628970] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7874-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.628973] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.629104] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7874-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.684376] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.686935] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886E-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:17.686961] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886E-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.687795] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4D-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.690439] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4D-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:17.690458] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4D-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.691580] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7664-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.692478] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6895-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.694485] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7664-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.696313] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7664-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.696407] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7664-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:17.703809] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDA7-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.708469] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224176378593003, replayable_scn:1755224176378593003, readable_scn:1755224176378593003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224177705622, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224176378593003, replayable_scn:1755224176378593003, readable_scn:1755224176378593003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224177705622, ora_rowscn_=1755224177625615000, content_changed=true) [2025-08-15 10:16:17.708648] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:17.708661] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:17.708666] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:17.752794] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDB6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.752918] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF080-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.806091] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:17.806133] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36CC-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224177805577000, start_id=1755224177805577000, end_id=1755224177805577001) [2025-08-15 10:16:17.806837] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36CC-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224177805577000, v:0}, tenant_id=1011) [2025-08-15 10:16:17.807376] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224177806010, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:16:17.807401] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224182807399, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:16:17.807491] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC64E-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.808587] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224177807220, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:16:17.808608] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224182808606, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:16:17.850544] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D36-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.924312] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB4-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224177923396000, start_id=1755224177923396000, end_id=1755224177923396001) [2025-08-15 10:16:17.925026] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB4-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224177923396000, v:0}, tenant_id=1013) [2025-08-15 10:16:17.926797] INFO [RS] finish (ob_mview_maintenance_task.cpp:280) [727135][T1016_TimerWK3_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=12] mview maintenace task finish(tenant_id_=1016, round_=63409, status_=100, error_code_=0, cost_us_=20001412, prepare_cost_us_=0, gc_mview_cost_us_=1245, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:16:17.958683] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B390-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:17.960953] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C7-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.963528] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C7-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:17.963543] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C7-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.969395] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:17.970651] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224177969317, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:16:17.970677] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224182970675, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:16:17.981853] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8893-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.984383] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8893-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:17.984399] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8893-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:17.993948] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5901-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:17.994648] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9E4-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.014216] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5A-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.017265] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5A-0-0] [lt=99] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:18.017283] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.026551] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E5-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:18.029051] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E5-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:18.044141] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8395-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.046561] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8395-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.048458] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8395-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.048547] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8395-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.055091] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:18.056323] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224178055010, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:16:18.056347] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224183056345, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:16:18.104121] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B4-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224178103141000, start_id=1755224178103141000, end_id=1755224178103141001) [2025-08-15 10:16:18.104902] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B4-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224178103141000, v:0}, tenant_id=1012) [2025-08-15 10:16:18.104924] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B4-0-0] [lt=14] zone_merge_mgr is already loaded(tenant_id=1012, global_merge_info={tenant_id:1012, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194400967887, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194695172923, need_update:false}}, zone_merge_infos=[{tenant_id:1012, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194695173510, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194400994787, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1012, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194695175345, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401001136, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1012, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194695177103, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401004882, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:16:18.106420] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58FB-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.123801] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463BC-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224178123091000, start_id=1755224178123091000, end_id=1755224178123091001) [2025-08-15 10:16:18.124618] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463BC-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224178123091000, v:0}, tenant_id=1) [2025-08-15 10:16:18.138622] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0DC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.149765] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E907-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:18.151001] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF07D-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.153480] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E907-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:18.156292] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD527C-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:18.158786] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD527C-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:18.187850] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D34-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.202314] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9E9-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.245493] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD531B-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:18.245504] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B2-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:18.247983] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD531B-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:18.247990] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B2-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:18.284963] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95CC5-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}) [2025-08-15 10:16:18.285005] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95CC5-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:16:18.312795] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC1-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:18.315310] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC1-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:18.352381] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C5-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:18.354951] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C5-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:18.443672] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6893-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.448524] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227778447362}) [2025-08-15 10:16:18.448542] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:18.448553] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:18.454424] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227778453278}) [2025-08-15 10:16:18.454438] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:18.454446] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=5][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:18.455624] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227778454456}) [2025-08-15 10:16:18.455637] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:18.455647] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:18.462408] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:18.462427] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:18.462436] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:18.495907] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8C-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.497717] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A34-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.498456] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8C-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:18.498479] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8C-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.500267] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A34-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:18.500285] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A34-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.503442] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0DA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.503489] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E749-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.512072] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF228-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.534418] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:18.534429] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:18.534437] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:18.535008] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:18.535022] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:18.535028] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:18.540432] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:18.540445] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:18.540450] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:18.541059] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5840-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81054688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81445312GB}) [2025-08-15 10:16:18.541101] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5840-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:16:18.546578] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:18.546593] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:18.546605] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:18.546621] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1008, cost_time_us=197, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:18.551913] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:18.551924] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:18.551930] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:18.584427] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8273-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1005, thread_idx=0) [2025-08-15 10:16:18.585691] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1005, only_for_display=false) [2025-08-15 10:16:18.586948] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=16] start try ls disaster recovery(ls_status_info={tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:18.600790] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.601966] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.603130] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.603154] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1005, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:18.603167] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.603179] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1005, ls_id:{id:1}, replicas:[{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:18.603282] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=80] build ls locality stat map(tenant_id=1005, ls_id={id:1}) [2025-08-15 10:16:18.603290] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.603304] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.603315] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.603327] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:18.603405] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.603422] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.603432] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.603437] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.603446] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3813, tenant_id=1005, ls_id={id:1}) [2025-08-15 10:16:18.603456] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:18.604524] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18834, tenant_id=1005, acc_dr_task=0) [2025-08-15 10:16:18.605759] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1006, only_for_display=false) [2025-08-15 10:16:18.606953] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:18.613202] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:18.614453] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224178613118, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:16:18.614479] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224183614477, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:16:18.620607] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.621753] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.622897] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.622920] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:18.622936] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.622954] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=14] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1}, replicas:[{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:18.623077] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=96] build ls locality stat map(tenant_id=1006, ls_id={id:1}) [2025-08-15 10:16:18.623085] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.623098] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.623106] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.623117] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:18.623194] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.623209] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.623219] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.623224] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.623233] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3776, tenant_id=1006, ls_id={id:1}) [2025-08-15 10:16:18.623243] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:18.624323] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:18.625765] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7707-0-0] [lt=9] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224178381753003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224178381753003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224178381753003, v:0}}], last_dump_ts_=1755224173594411, force_dump=false) [2025-08-15 10:16:18.625785] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7707-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.628825] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7707-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224178381753003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224178381753003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224178381753003, v:0}}], last_dump_ts_=1755224173597411, force_dump=false) [2025-08-15 10:16:18.628843] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7707-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.629685] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.629703] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E6-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.629911] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.630065] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7875-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.630855] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7707-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.630960] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7707-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.632759] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06E-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.632926] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E6-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.633036] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.633076] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7875-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.634974] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E6-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.635035] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.635096] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E6-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.635143] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3C-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.635331] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.635410] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06E-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.635536] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7875-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.635623] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7875-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.645296] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.646457] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.647596] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.647618] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:18.647635] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.647649] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1001}, replicas:[{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:18.647758] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=86] build ls locality stat map(tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:16:18.647767] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.647779] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.647790] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.647802] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:18.647880] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.647894] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.647904] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.647913] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.647922] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3776, tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:16:18.647933] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:18.649010] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:18.669674] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.670822] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.671969] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.671991] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:18.672006] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.672015] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1002}, replicas:[{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:18.672115] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=79] build ls locality stat map(tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:16:18.672123] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.672131] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.672137] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.672145] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:18.672220] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.672231] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.672238] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.672245] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.672252] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3703, tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:16:18.672259] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:18.673009] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF2-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1, thread_idx=0) [2025-08-15 10:16:18.673314] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:18.674249] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=17] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-08-15 10:16:18.675457] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:18.685582] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [438235][T1005_MergeSche][T1005][YB420ACB0122-000633820772DF2C-0-0] [lt=63] zone_merge_mgr is already loaded(tenant_id=1005, global_merge_info={tenant_id:1005, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401690636, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194754861110, need_update:false}}, zone_merge_infos=[{tenant_id:1005, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194754863631, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401763272, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1005, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194754867140, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401768897, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1005, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194754868967, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401771210, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:16:18.689341] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.689708] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886F-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.690497] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.691652] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.691674] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:18.691692] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.691703] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:18.691804] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=79] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-08-15 10:16:18.691814] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.691826] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.691835] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.691842] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:18.691919] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.691929] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.691936] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.691947] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.691957] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3792, tenant_id=1, ls_id={id:1}) [2025-08-15 10:16:18.691968] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:18.692262] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886F-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:18.692281] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D886F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.692552] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6896-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.693108] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18859, tenant_id=1, acc_dr_task=0) [2025-08-15 10:16:18.693275] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4E-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.693969] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=9] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.695112] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.695829] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4E-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:18.695847] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4E-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.696274] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:18.696296] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:18.696308] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.696320] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1003}, replicas:[{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:18.696425] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=83] build ls locality stat map(tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:16:18.696432] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.696444] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.696452] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:18.696463] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:18.696539] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.696554] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.696568] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=10] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.696578] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:18.696583] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3746, tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:16:18.696593] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:18.697426] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7665-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.697784] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93246, tenant_id=1006, acc_dr_task=0) [2025-08-15 10:16:18.700454] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7665-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.702322] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7665-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.702407] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7665-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:18.703887] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDA8-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.708576] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:18.708588] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:18.708594] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:18.752869] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDB7-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.752992] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF081-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.759763] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5C-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224178759422000, start_id=1755224178759422000, end_id=1755224178759422001) [2025-08-15 10:16:18.760509] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5C-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224178759422000, v:0}, tenant_id=1016) [2025-08-15 10:16:18.760532] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5C-0-0] [lt=14] zone_merge_mgr is already loaded(tenant_id=1016, global_merge_info={tenant_id:1016, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402661472, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194765625452, need_update:false}}, zone_merge_infos=[{tenant_id:1016, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194765627764, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402681290, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1016, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194765631279, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402686302, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1016, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194765633133, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402690030, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:16:18.850616] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D37-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.933563] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224178548949, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224176378593002, replayable_scn:1755224176378593002, readable_scn:1755224176378593002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:18.960163] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B391-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:18.966275] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C8-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.968800] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C8-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:18.968818] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C8-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.987054] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8894-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.989621] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8894-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:18.989638] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8894-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:18.994026] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5902-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:18.994723] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9E5-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.019936] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5B-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.022432] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5B-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:19.022450] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.034428] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E6-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:19.036890] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:19.049521] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8396-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.052005] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8396-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224178882469000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224178882469000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224178882469000, v:0}}], last_dump_ts_=1755224174023163, force_dump=false) [2025-08-15 10:16:19.052033] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8396-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.054133] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8396-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.054223] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8396-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.106470] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58FC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.138696] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0DD-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.151074] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF07E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.158925] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E908-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:19.161476] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E908-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:19.164114] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD527D-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:19.166615] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD527D-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:19.175969] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64F-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224179175848000, start_id=1755224179175848000, end_id=1755224179175848001) [2025-08-15 10:16:19.176705] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64F-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224179175848000, v:0}, tenant_id=1006) [2025-08-15 10:16:19.187923] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D35-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.202387] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9EA-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.240218] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A8-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224179239988000, start_id=1755224179239988000, end_id=1755224179239988001) [2025-08-15 10:16:19.241057] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A8-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224179239988000, v:0}, tenant_id=1008) [2025-08-15 10:16:19.253416] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD531C-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:19.253425] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B3-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:19.255891] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD531C-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:19.255896] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B3-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:19.320602] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC2-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:19.323065] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC2-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:19.360379] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C6-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:19.362927] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C6-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:19.417094] INFO [RS] gc_mview (ob_mview_maintenance_task.cpp:267) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=15] mview maintenance task gc mview success(tenant_id_=1, round_=121920, gc_mview_cost_us_=1213, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:16:19.443748] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6894-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.462453] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:19.462466] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:19.462472] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:19.463779] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:19.473917] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A08F3-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}) [2025-08-15 10:16:19.473965] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A08F3-0-0] [lt=31] [DDL] execute ddl like stmt(ret=0, cost=50, ddl_arg=NULL) [2025-08-15 10:16:19.501242] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8D-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.502982] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A35-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.503517] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0DB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.503564] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E74A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.503787] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8D-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:19.503806] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8D-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.505523] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A35-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:19.505540] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A35-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.534466] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:19.534478] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:19.534484] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:19.535140] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:19.535153] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:19.535160] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:19.535771] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:19.540324] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224177380580002, replayable_scn:1755224177380580002, readable_scn:1755224177380580002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224179537625, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224177380580002, replayable_scn:1755224177380580002, readable_scn:1755224177380580002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224179537625, ora_rowscn_=1755224178627893000, content_changed=true) [2025-08-15 10:16:19.540500] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:19.540517] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:19.540522] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:19.546610] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:19.546623] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:19.546628] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:19.551984] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:19.551995] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:19.552001] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:19.553276] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:19.556669] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E65-0-0] [lt=12] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1003, thread_idx=0) [2025-08-15 10:16:19.557934] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=17] start try tenant disaster recovery(tenant_id=1003, only_for_display=false) [2025-08-15 10:16:19.559165] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=21] start try ls disaster recovery(ls_status_info={tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:19.573051] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.574202] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.575358] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.575384] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1003, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:19.575397] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.575410] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1003, ls_id:{id:1}, replicas:[{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:19.575517] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=83] build ls locality stat map(tenant_id=1003, ls_id={id:1}) [2025-08-15 10:16:19.575528] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.575542] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.575549] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.575560] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:19.575637] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.575652] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.575661] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.575667] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.575674] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3798, tenant_id=1003, ls_id={id:1}) [2025-08-15 10:16:19.575684] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:19.576765] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18831, tenant_id=1003, acc_dr_task=0) [2025-08-15 10:16:19.578003] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1004, only_for_display=false) [2025-08-15 10:16:19.579194] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:19.592988] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.594151] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.595328] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.595351] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:19.595367] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.595379] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:19.595485] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=83] build ls locality stat map(tenant_id=1004, ls_id={id:1}) [2025-08-15 10:16:19.595495] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.595512] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.595523] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.595535] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:19.595616] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.595630] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.595640] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.595651] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.595657] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3842, tenant_id=1004, ls_id={id:1}) [2025-08-15 10:16:19.595667] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:19.596739] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:19.617576] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.618751] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.619893] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.619915] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:19.619929] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.619943] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:19.620047] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=82] build ls locality stat map(tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:16:19.620056] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.620066] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.620074] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.620083] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:19.620157] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.620169] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.620176] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=4] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.620182] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.620189] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3765, tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:16:19.620196] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:19.621259] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:19.632031] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7708-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.633744] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:19.634970] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7708-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.634996] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224179633669, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:16:19.635020] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224184635018, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:16:19.636099] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3D-0-0] [lt=10] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224179382994002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224179382994002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224179382994002, v:0}}], last_dump_ts_=1755224174605107, force_dump=false) [2025-08-15 10:16:19.636113] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E7-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.636124] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3D-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.636338] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06F-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.636553] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7876-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224179382994002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224179382994002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224179382994002, v:0}}], last_dump_ts_=1755224174605101, force_dump=false) [2025-08-15 10:16:19.636574] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7876-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.636906] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7708-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.637043] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7708-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.639102] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E7-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.639160] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3D-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224179382994002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224179382994002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224179382994002, v:0}}], last_dump_ts_=1755224174608198, force_dump=false) [2025-08-15 10:16:19.639182] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3D-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.639304] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06F-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.639578] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7876-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.641053] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E7-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.641185] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E7-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.641190] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3D-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.641251] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.641279] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.641370] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC06F-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.641509] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7876-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.641593] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7876-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.642232] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.643398] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.644546] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.644569] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:19.644587] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.644598] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1002}, replicas:[{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:19.644703] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=82] build ls locality stat map(tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:16:19.644711] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.644724] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.644732] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.644744] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:19.644821] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.644835] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.644845] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.644850] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.644858] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3825, tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:16:19.644868] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:19.645964] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:19.666689] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.667842] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.669009] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.669040] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:19.669060] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.669070] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1003}, replicas:[{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:19.669181] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=86] build ls locality stat map(tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:16:19.669189] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.669201] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.669211] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.669223] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:19.669300] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.669314] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.669323] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.669333] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.669339] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3804, tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:16:19.669349] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:19.670422] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=8] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93642, tenant_id=1004, acc_dr_task=0) [2025-08-15 10:16:19.692637] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6897-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.695318] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8870-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.697844] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8870-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:19.697862] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8870-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.698581] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4F-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.701204] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4F-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:19.701222] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A4F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.703404] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7666-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224179501864000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224179501864001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224179501864001, v:0}}], last_dump_ts_=1755224174673267, force_dump=false) [2025-08-15 10:16:19.703430] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7666-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.703963] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDA9-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.706376] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7666-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224179501864000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224179501864001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224179501864001, v:0}}], last_dump_ts_=1755224174676278, force_dump=false) [2025-08-15 10:16:19.706397] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7666-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.708240] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7666-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.708421] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7666-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:19.708746] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:19.708759] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:19.708764] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:19.710104] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:19.734183] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E8-0-0] [lt=18] get number(ret=0, service_type_=0, range=1, base_id=1755224179733446000, start_id=1755224179733446000, end_id=1755224179733446001) [2025-08-15 10:16:19.734876] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E8-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224179733446000, v:0}, tenant_id=1003) [2025-08-15 10:16:19.752943] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDB8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.753069] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF082-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.812451] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC64F-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.850692] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D38-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.875887] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C5-0-0] [lt=57] get number(ret=0, service_type_=0, range=1, base_id=1755224179875373000, start_id=1755224179875373000, end_id=1755224179875373001) [2025-08-15 10:16:19.876711] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C5-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224179875373000, v:0}, tenant_id=1004) [2025-08-15 10:16:19.892338] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BE-0-0] [lt=13] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1017, thread_idx=0) [2025-08-15 10:16:19.893593] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1017, only_for_display=false) [2025-08-15 10:16:19.894818] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:19.908586] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.909740] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.910878] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.910900] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1017, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:19.910914] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.910923] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1017, ls_id:{id:1}, replicas:[{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:19.911031] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=83] build ls locality stat map(tenant_id=1017, ls_id={id:1}) [2025-08-15 10:16:19.911042] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.911052] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.911060] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.911067] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:19.911142] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.911154] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.911161] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.911168] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.911175] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3760, tenant_id=1017, ls_id={id:1}) [2025-08-15 10:16:19.911183] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:19.912251] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18658, tenant_id=1017, acc_dr_task=0) [2025-08-15 10:16:19.913448] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1018, only_for_display=false) [2025-08-15 10:16:19.914645] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:19.928351] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.929507] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.930660] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.930682] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:19.930699] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.930710] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1}, replicas:[{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:19.930813] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=79] build ls locality stat map(tenant_id=1018, ls_id={id:1}) [2025-08-15 10:16:19.930821] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.930834] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.930846] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.930854] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:19.930946] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=63] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.930960] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.930970] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.930977] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.930985] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3798, tenant_id=1018, ls_id={id:1}) [2025-08-15 10:16:19.930996] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:19.932052] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:19.952758] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.953887] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.955044] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.955067] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:19.955084] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.955095] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1001}, replicas:[{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:19.955199] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=81] build ls locality stat map(tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:16:19.955212] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.955225] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.955236] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.955247] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:19.955324] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.955338] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.955347] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.955353] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.955361] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3745, tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:16:19.955372] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:19.956483] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:19.961558] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B392-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:19.971531] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C9-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.974050] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C9-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:19.974072] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95C9-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.977230] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.978383] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.979701] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:19.979732] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:19.979755] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=15] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.979770] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1002}, replicas:[{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:19.979959] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=148] build ls locality stat map(tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:16:19.979977] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=12] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.979997] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=15] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.980014] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=13] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:19.980031] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:19.980179] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=112] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.980201] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=11] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.980215] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=10] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.980229] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=11] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:19.980242] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=10] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=4150, tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:16:19.980257] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=12] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:19.981560] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:19.992328] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8895-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:19.994059] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5903-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.994794] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9E6-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:19.994931] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8895-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:19.994956] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8895-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.002400] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:20.003553] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:20.004696] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:20.004719] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:20.004736] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:20.004747] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1003}, replicas:[{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:20.004849] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:16:20.004857] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:20.004871] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:20.004881] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:20.004891] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:20.004975] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:20.004990] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:20.004999] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:20.005004] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:20.005012] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3774, tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:16:20.005021] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:20.006096] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93832, tenant_id=1018, acc_dr_task=0) [2025-08-15 10:16:20.010681] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=6] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:20.012573] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [17739][T1_TimerWK2_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=12] mview_mds_op(tx_count=0) [2025-08-15 10:16:20.012584] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [17739][T1_TimerWK2_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=8] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=818) [2025-08-15 10:16:20.025232] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5C-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.027780] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5C-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:20.027799] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5C-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.042272] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E7-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:20.044796] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E7-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:20.055291] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8397-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224179790175000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224179790175000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224179790175000, v:0}}], last_dump_ts_=1755224175026212, force_dump=false) [2025-08-15 10:16:20.055316] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8397-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.057794] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8397-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.059761] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8397-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.059842] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8397-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.106545] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58FD-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.138767] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0DE-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.151147] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF07F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.166761] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E909-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:20.169280] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E909-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:20.171897] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD527E-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:20.174408] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD527E-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:20.188004] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D36-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.202465] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9EB-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.206243] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C6-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224180205805000, start_id=1755224180205805000, end_id=1755224180205805001) [2025-08-15 10:16:20.206947] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C6-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224180205805000, v:0}, tenant_id=1007) [2025-08-15 10:16:20.261330] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD531D-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:20.261344] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B4-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:20.263802] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B4-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:20.263804] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD531D-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:20.285491] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D74-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}) [2025-08-15 10:16:20.285539] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D74-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=53, ddl_arg=NULL) [2025-08-15 10:16:20.328531] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC3-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:20.331080] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC3-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:20.368392] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C7-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:20.370918] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C7-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:20.443821] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6895-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.462398] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=12] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224178881451003, replayable_scn:1755224178881451003, readable_scn:1755224178881451003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224180459398, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224178881451003, replayable_scn:1755224178881451003, readable_scn:1755224178881451003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224180459398, ora_rowscn_=1755224180056412000, content_changed=true) [2025-08-15 10:16:20.462580] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:20.462591] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:20.462597] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:20.503592] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0DC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.503642] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E74B-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.506537] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8E-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.508181] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A36-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.509033] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8E-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:20.509048] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8E-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.510688] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A36-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:20.510703] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A36-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.512344] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF229-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.534931] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224178386986002, replayable_scn:1755224178386986002, readable_scn:1755224178386986002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224180530978, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224178386986002, replayable_scn:1755224178386986002, readable_scn:1755224178386986002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224180530978, ora_rowscn_=1755224179705047000, content_changed=true) [2025-08-15 10:16:20.535056] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224178380699002, replayable_scn:1755224178380699002, readable_scn:1755224178380699002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224180532032, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224178380699002, replayable_scn:1755224178380699002, readable_scn:1755224178380699002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224180532032, ora_rowscn_=1755224179637755000, content_changed=true) [2025-08-15 10:16:20.535147] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=30] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:20.535165] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:20.535178] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:20.535219] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:20.535233] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:20.535244] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:20.541147] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:20.541165] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:20.541173] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:20.541547] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAAE-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81445312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81835938GB}) [2025-08-15 10:16:20.541588] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAAE-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:16:20.542562] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:20.546478] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224178380699002, replayable_scn:1755224178380699002, readable_scn:1755224178380699002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224180543522, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224178380699002, replayable_scn:1755224178380699002, readable_scn:1755224178380699002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224180543522, ora_rowscn_=1755224179637755000, content_changed=true) [2025-08-15 10:16:20.546667] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:20.546679] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:20.546684] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:20.551850] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224178380699002, replayable_scn:1755224178380699002, readable_scn:1755224178380699002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224180548790, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224178380699002, replayable_scn:1755224178380699002, readable_scn:1755224178380699002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224180548790, ora_rowscn_=1755224179637755000, content_changed=true) [2025-08-15 10:16:20.552017] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:20.552027] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:20.552034] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:20.606297] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [1487816][T1003_MergeSche][T1003][YB420ACB0122-0006337FE3A2DCAD-0-0] [lt=65] zone_merge_mgr is already loaded(tenant_id=1003, global_merge_info={tenant_id:1003, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403772779, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194756784394, need_update:false}}, zone_merge_infos=[{tenant_id:1003, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194756785330, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403861743, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1003, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194756787126, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403865394, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1003, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194756788906, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403869575, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:16:20.628803] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:20.630059] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224180628728, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:16:20.630083] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224185630082, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:16:20.638030] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7709-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.641101] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7709-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.642151] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E8-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.642208] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.642327] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC070-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.642582] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7877-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.643182] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7709-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.643284] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7709-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.645136] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E8-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.645244] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3E-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.645490] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC070-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.645494] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7877-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.647297] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3E-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.647386] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.647469] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC070-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.647474] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7877-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.647556] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC070-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.647557] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7877-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.647716] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E8-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.647894] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E8-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.692711] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6898-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.700604] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8871-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.703232] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8871-0-0] [lt=71] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:20.703251] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8871-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.703935] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A50-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.704041] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDAA-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.706469] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A50-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:20.706487] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A50-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.708586] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224179382994002, replayable_scn:1755224179382994002, readable_scn:1755224179382994002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224180705593, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224179382994002, replayable_scn:1755224179382994002, readable_scn:1755224179382994002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224180705593, ora_rowscn_=1755224180644583000, content_changed=true) [2025-08-15 10:16:20.708774] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:20.708787] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:20.708792] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:20.709447] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7667-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.712379] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7667-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.714241] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7667-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.714334] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7667-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:20.728281] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [3414201][T1006_TimerWK4_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:20.753016] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDB9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.753140] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF083-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.850767] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D39-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.953440] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:20.954690] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224180953361, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:16:20.954713] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224185954711, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:16:20.962984] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B393-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:20.976775] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CA-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.979953] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CA-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:20.979974] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CA-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:20.994130] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5904-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.994873] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9E7-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:20.997625] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8896-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.000230] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8896-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:21.000249] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8896-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.030574] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5D-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.033115] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5D-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:21.033132] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5D-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.050195] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E8-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:21.051376] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E8-0-0] [lt=12] log restore source is empty, just skip(ret=-4018, user_tenant_id=1004) [2025-08-15 10:16:21.053287] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B12-0-0] [lt=12] [RESTORE] restore scheduler quit [2025-08-15 10:16:21.054046] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E8-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:21.054546] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B12-0-0] [lt=12] get targets(ret=0, jobs=[], exec_tenant_id=1005, sql=select * from __all_recover_table_job) [2025-08-15 10:16:21.055728] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B13-0-0] [lt=13] get targets(ret=0, jobs=[], exec_tenant_id=1005, sql=select * from __all_import_table_job) [2025-08-15 10:16:21.055752] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B14-0-0] [lt=13][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224181055746, ref_timestamp=1755224121052038, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:16:21.055832] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B15-0-0] [lt=10] [RESTORE] restore scheduler start [2025-08-15 10:16:21.055839] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B16-0-0] [lt=6] [RESTORE] try process restore job [2025-08-15 10:16:21.056971] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B16-0-0] [lt=3] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:16:21.060851] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8398-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.063456] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8398-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.065453] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8398-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.065556] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8398-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.067295] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:21.068581] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224181058822, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:16:21.068607] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224186068605, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:16:21.068622] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224186068619, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:21.106620] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58FE-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.138841] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0DF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.151221] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF080-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.174564] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E90A-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:21.177048] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E90A-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:21.179632] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD527F-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:21.182084] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD527F-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:21.188047] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D37-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.196265] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:21.197502] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224181196193, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:16:21.197528] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224186197526, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:16:21.198441] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:21.199664] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224181198370, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:16:21.199686] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224186199685, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:21.202537] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9EC-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.269160] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B5-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:21.269161] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD531E-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:21.271648] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD531E-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:21.271653] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B5-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:21.336454] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC4-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:21.338924] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC4-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:21.376200] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C8-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:21.378722] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C8-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:21.419174] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E53-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224181418656000, start_id=1755224181418656000, end_id=1755224181418656001) [2025-08-15 10:16:21.419977] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E53-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224181418656000, v:0}, tenant_id=1018) [2025-08-15 10:16:21.443896] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6896-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.459967] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227781458784}) [2025-08-15 10:16:21.459986] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:21.459995] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:21.462556] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:21.462576] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:21.462583] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:21.465906] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227781464741}) [2025-08-15 10:16:21.465923] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:21.465936] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:21.467125] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227781465952}) [2025-08-15 10:16:21.467140] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:21.467154] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:21.474386] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D37-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}) [2025-08-15 10:16:21.474434] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D37-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=54, ddl_arg=NULL) [2025-08-15 10:16:21.503631] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0DD-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.503715] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E74C-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.511706] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8F-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.513758] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A37-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.514196] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8F-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:21.514213] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF8F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.516412] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A37-0-0] [lt=91] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:21.516428] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A37-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.534677] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:21.534691] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:21.534697] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:21.535203] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:21.535214] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:21.535221] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:21.540612] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:21.540624] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:21.540635] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:21.546708] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:21.546723] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:21.546729] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:21.546736] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1008, cost_time_us=180, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:21.552146] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:21.552164] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:21.552171] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:21.644237] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.647161] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.648254] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3F-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.648420] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7878-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.648525] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC071-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.648875] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E9-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224181384988003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224181384988003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224181384988003, v:0}}], last_dump_ts_=1755224176617373, force_dump=false) [2025-08-15 10:16:21.648899] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E9-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.649133] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.649239] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.651439] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3F-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.651591] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC071-0-0] [lt=16] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224181384988002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224181384988002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224181384988002, v:0}}], last_dump_ts_=1755224176620570, force_dump=false) [2025-08-15 10:16:21.651618] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC071-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.651622] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7878-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.651735] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E9-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224181384988003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224181384988003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224181384988003, v:0}}], last_dump_ts_=1755224176620499, force_dump=false) [2025-08-15 10:16:21.651757] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E9-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.653385] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.653487] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C3F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.653697] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E9-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.653735] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7878-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.653758] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC071-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.653787] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413E9-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.653825] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7878-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.653842] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC071-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.692785] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6899-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.704116] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDAB-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.706582] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8872-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.708866] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:21.708880] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:21.708885] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:21.709186] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8872-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:21.709206] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8872-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.709181] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A51-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.711709] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A51-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:21.711728] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A51-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.715260] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7668-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.718282] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7668-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.720166] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7668-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.720246] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7668-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:21.753091] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDBA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.753213] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF084-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.787725] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709536][T1014_TimerWK3_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:21.813559] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA3-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224181812827000, start_id=1755224181812827000, end_id=1755224181812827001) [2025-08-15 10:16:21.814401] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA3-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224181812827000, v:0}, tenant_id=1014) [2025-08-15 10:16:21.817632] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC650-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.837636] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:21.838883] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224181837556, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:16:21.838910] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224186838908, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:16:21.839617] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:16:21.843874] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D5-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1015, thread_idx=0) [2025-08-15 10:16:21.845150] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1015, only_for_display=false) [2025-08-15 10:16:21.846361] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:21.850842] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D3A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.860324] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.861499] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.862680] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.862703] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1015, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:21.862721] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.862733] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1015, ls_id:{id:1}, replicas:[{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:21.862835] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=80] build ls locality stat map(tenant_id=1015, ls_id={id:1}) [2025-08-15 10:16:21.862844] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.862857] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.862868] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.862876] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:21.862960] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.862978] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.862988] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.862993] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.863001] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3822, tenant_id=1015, ls_id={id:1}) [2025-08-15 10:16:21.863008] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:21.864068] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18918, tenant_id=1015, acc_dr_task=0) [2025-08-15 10:16:21.865313] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1016, only_for_display=false) [2025-08-15 10:16:21.866523] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:21.880898] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.882048] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.883246] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.883269] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:21.883286] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.883297] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1}, replicas:[{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:21.883398] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=79] build ls locality stat map(tenant_id=1016, ls_id={id:1}) [2025-08-15 10:16:21.883406] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.883419] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.883430] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.883437] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:21.883513] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.883527] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.883536] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.883541] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.883550] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3826, tenant_id=1016, ls_id={id:1}) [2025-08-15 10:16:21.883559] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:21.884693] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:21.905518] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.906679] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.907818] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.907840] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:21.907857] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.907867] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1001}, replicas:[{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:21.907977] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=86] build ls locality stat map(tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:16:21.907985] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.907998] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.908008] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.908018] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:21.908095] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.908108] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.908118] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.908128] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.908138] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3776, tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:16:21.908149] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:21.909223] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:21.929998] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.931149] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.932288] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.932310] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:21.932324] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.932335] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1002}, replicas:[{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:21.932437] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=80] build ls locality stat map(tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:16:21.932445] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.932458] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.932468] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.932475] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:21.932552] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.932570] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.932579] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.932584] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.932592] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3747, tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:16:21.932604] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:21.933712] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:21.935161] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=17] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224181549081, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224179382994002, replayable_scn:1755224179382994002, readable_scn:1755224179382994002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:21.954524] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.955684] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.956848] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:21.956870] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:21.956881] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=6] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.956890] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1003}, replicas:[{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:21.956997] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=84] build ls locality stat map(tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:16:21.957008] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.957019] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.957027] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:21.957036] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:21.957119] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.957131] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.957138] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.957145] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:21.957151] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3779, tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:16:21.957160] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:21.958271] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94189, tenant_id=1016, acc_dr_task=0) [2025-08-15 10:16:21.964376] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B394-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:21.982676] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CB-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.985180] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CB-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:21.985198] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CB-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:21.994206] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5905-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:21.994950] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9E8-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.002934] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8897-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.005471] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8897-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:22.005490] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8897-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.035910] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.038441] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5E-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:22.038457] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5E-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.059345] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E9-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:22.061850] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38E9-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:22.066770] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8399-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.069813] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8399-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.071737] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8399-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.071854] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8399-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.106697] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58FF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.138915] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0E0-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.151292] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF081-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.173898] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727134][T1016_TimerWK2_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=18] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:22.182388] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E90B-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:22.184867] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E90B-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:22.187250] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5280-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:22.188122] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D38-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.189716] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5280-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:22.202609] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9ED-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.213316] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727957][T1018_TimerWK1_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:22.267650] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619488][T1012_TimerWK3_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=44] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:22.276965] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD531F-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:22.276996] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B6-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:22.279490] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD531F-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:22.279504] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B6-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:22.286007] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95CC7-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}) [2025-08-15 10:16:22.286049] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95CC7-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:16:22.327949] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487604][T1004_TimerWK1_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:22.342545] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:22.343798] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224182342468, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:16:22.343822] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224187343820, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:16:22.344201] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC5-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:22.346667] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC5-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:22.383919] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C9-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:22.386392] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62C9-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:22.434151] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:22.435385] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224182434075, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:16:22.435408] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224187435406, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:16:22.438575] INFO [RS] prepare (ob_mview_maintenance_task.cpp:176) [475910][T1008_TimerWK2_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=10] mview maintenance task prepare success(tenant_id_=1008, round_=105551, prepare_cost_us_=0) [2025-08-15 10:16:22.443969] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6897-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.462663] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:22.462676] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:22.462681] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:22.503706] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0DE-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.503784] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E74D-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.512602] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF22A-0-0] [lt=10] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.516972] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF90-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.519096] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A38-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.519496] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF90-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:22.519516] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF90-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.521652] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A38-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:22.521669] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A38-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.534643] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:22.534655] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:22.534663] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:22.535235] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:22.535250] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:22.535261] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:22.536552] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:22.540513] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224180384644003, replayable_scn:1755224180384644003, readable_scn:1755224180384644003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224182538369, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224180384644003, replayable_scn:1755224180384644003, readable_scn:1755224180384644003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224182538369, ora_rowscn_=1755224181645748000, content_changed=true) [2025-08-15 10:16:22.540697] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:22.540715] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:22.540726] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:22.542015] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6AA5-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81835938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82226562GB}) [2025-08-15 10:16:22.542055] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6AA5-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2025-08-15 10:16:22.546802] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:22.546818] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:22.546825] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:22.548184] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:22.552109] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:22.552122] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:22.552128] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:22.650281] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770B-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.653458] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.654423] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C40-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.654725] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EA-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.654724] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC072-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224182386870002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224182386870002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224182386870002, v:0}}], last_dump_ts_=1755224177623612, force_dump=false) [2025-08-15 10:16:22.654749] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC072-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.654763] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7879-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.655437] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770B-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.655515] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.657466] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C40-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.657724] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC072-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.657762] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EA-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.657807] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7879-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224182386870003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224182386870003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224182386870003, v:0}}], last_dump_ts_=1755224177626908, force_dump=false) [2025-08-15 10:16:22.657834] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7879-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.659541] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C40-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.659646] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C40-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.659772] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC072-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.659847] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EA-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.659851] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7879-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.659876] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC072-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.659945] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EA-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.659956] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7879-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.692862] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C689A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.704193] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDAC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.708856] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:22.708870] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:22.708877] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:22.711946] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8873-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.714537] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8873-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:22.714552] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8873-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.714518] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A52-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.717223] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A52-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:22.717242] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A52-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.721402] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7669-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.724384] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7669-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.726347] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7669-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.726459] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7669-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:22.753167] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDBB-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.753285] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF085-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.807473] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:22.808718] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224182807399, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:16:22.808741] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224187808738, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:16:22.809928] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224182808606, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:16:22.809949] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224187809948, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:16:22.850916] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D3B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.932108] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EC-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1013, thread_idx=0) [2025-08-15 10:16:22.933374] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1013, only_for_display=false) [2025-08-15 10:16:22.934621] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:22.948528] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:22.949709] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:22.950898] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:22.950920] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1013, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:22.950943] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.950955] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1013, ls_id:{id:1}, replicas:[{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:22.951058] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=80] build ls locality stat map(tenant_id=1013, ls_id={id:1}) [2025-08-15 10:16:22.951066] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:22.951080] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:22.951086] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:22.951097] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:22.951174] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.951188] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.951198] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.951203] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.951211] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3839, tenant_id=1013, ls_id={id:1}) [2025-08-15 10:16:22.951222] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:22.952289] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18916, tenant_id=1013, acc_dr_task=0) [2025-08-15 10:16:22.953507] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1014, only_for_display=false) [2025-08-15 10:16:22.954730] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:22.965769] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B395-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:22.968725] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:22.969901] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=28] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:22.970752] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:22.971060] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:22.971083] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:22.971099] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.971112] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1}, replicas:[{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:22.971213] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=79] build ls locality stat map(tenant_id=1014, ls_id={id:1}) [2025-08-15 10:16:22.971220] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:22.971233] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:22.971244] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:22.971255] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:22.971332] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.971346] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.971356] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.971365] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.971369] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3848, tenant_id=1014, ls_id={id:1}) [2025-08-15 10:16:22.971379] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:22.972016] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224182970675, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:16:22.972041] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224187972039, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:16:22.972456] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:22.987915] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CC-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.990448] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CC-0-0] [lt=71] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:22.990469] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CC-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:22.993415] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:22.994283] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5906-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.994574] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:22.995024] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9E9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:22.995729] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:22.995752] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:22.995769] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.995780] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1001}, replicas:[{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:22.995883] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=80] build ls locality stat map(tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:16:22.995892] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:22.995905] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:22.995916] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:22.995927] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:22.996011] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.996026] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.996036] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.996041] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:22.996050] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3790, tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:16:22.996062] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:22.997148] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:23.009839] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8898-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.013621] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8898-0-0] [lt=100] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:23.013646] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8898-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.020634] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:23.021793] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:23.022957] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:23.022980] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:23.022998] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:23.023009] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1002}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:23.023112] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=80] build ls locality stat map(tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:16:23.023120] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:23.023134] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:23.023144] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:23.023156] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:23.023232] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:23.023246] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:23.023255] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:23.023262] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:23.023270] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3783, tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:16:23.023280] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:23.024360] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:23.041256] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5F-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.043805] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5F-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:23.043821] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB5F-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.045188] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:23.046326] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:23.047467] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:23.047491] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:23.047507] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:23.047518] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1003}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:23.047621] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=80] build ls locality stat map(tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:16:23.047629] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:23.047642] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:23.047649] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:23.047660] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:23.047737] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:23.047748] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:23.047756] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:23.047763] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:23.047772] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3722, tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:16:23.047782] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:23.048843] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=96539, tenant_id=1014, acc_dr_task=0) [2025-08-15 10:16:23.056418] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:23.057646] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224183056345, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:16:23.057669] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224188057667, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:16:23.067139] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38EA-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:23.069634] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38EA-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:23.072852] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839A-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.074983] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:340) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839A-0-0] [lt=13] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1014, ls_id:{id:1}, sync_scn:{val:1755224182887564001, v:0}, readable_scn:{val:1755224182887564001, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-08-15 10:16:23.075309] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839A-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.077264] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.077392] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.077428] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:459) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839A-0-0] [lt=6] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1755224180884410001, v:0}) [2025-08-15 10:16:23.106771] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5900-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.138989] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0E1-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.151366] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF082-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.188195] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D39-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.190247] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E90C-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:23.192757] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E90C-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:23.195142] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5281-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:23.198772] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5281-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:23.202682] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9EE-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.284867] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5320-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:23.284893] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B7-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:23.287391] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B7-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:23.287425] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5320-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:23.351984] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC6-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:23.353174] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC6-0-0] [lt=11] log restore source is empty, just skip(ret=-4018, user_tenant_id=1014) [2025-08-15 10:16:23.355640] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:23.391781] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62CA-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:23.394270] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62CA-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:23.444044] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6898-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.462515] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224181837024000, replayable_scn:1755224181837024000, readable_scn:1755224181837024000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224183459668, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224181837024000, replayable_scn:1755224181837024000, readable_scn:1755224181837024000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224183459668, ora_rowscn_=1755224183073716000, content_changed=true) [2025-08-15 10:16:23.462710] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:23.462722] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:23.462728] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:23.474869] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A096B-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.93164062GB}) [2025-08-15 10:16:23.474918] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A096B-0-0] [lt=30] [DDL] execute ddl like stmt(ret=0, cost=56, ddl_arg=NULL) [2025-08-15 10:16:23.503780] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0DF-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.503855] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E74E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.522219] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF91-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.524316] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A39-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.524724] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF91-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:23.524741] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF91-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.526853] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A39-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:23.526871] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A39-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.534699] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:23.534710] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:23.534719] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:23.535160] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224181384988003, replayable_scn:1755224181384988003, readable_scn:1755224181384988003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224183532145, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224181384988003, replayable_scn:1755224181384988003, readable_scn:1755224181384988003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224183532145, ora_rowscn_=1755224182655964000, content_changed=true) [2025-08-15 10:16:23.535341] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:23.535355] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:23.535361] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:23.540719] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:23.540734] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:23.540743] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:23.546707] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224181384988002, replayable_scn:1755224181384988002, readable_scn:1755224181384988002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224183543728, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224181384988002, replayable_scn:1755224181384988002, readable_scn:1755224181384988002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224183543728, ora_rowscn_=1755224182655964000, content_changed=true) [2025-08-15 10:16:23.546869] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:23.546879] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:23.546886] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:23.551973] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224181384988002, replayable_scn:1755224181384988002, readable_scn:1755224181384988002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224183548996, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224181384988002, replayable_scn:1755224181384988002, readable_scn:1755224181384988002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224183548996, ora_rowscn_=1755224182655964000, content_changed=true) [2025-08-15 10:16:23.552165] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:23.552179] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:23.552184] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:23.614548] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:23.615798] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224183614477, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:16:23.615822] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224188615820, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:16:23.656440] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770C-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224183388167003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224183388167003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224183388167003, v:0}}], last_dump_ts_=1755224178625764, force_dump=false) [2025-08-15 10:16:23.656466] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.659581] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770C-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224183388167003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224183388167003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224183388167003, v:0}}], last_dump_ts_=1755224178628824, force_dump=false) [2025-08-15 10:16:23.659600] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770C-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.660614] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C41-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.660792] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC073-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.660882] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787A-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.660904] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EB-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.661478] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770C-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.661586] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.663733] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C41-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.663756] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC073-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.663829] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EB-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.663871] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.665847] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.665850] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC073-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.665921] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C41-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.665949] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.665959] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC073-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.665957] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EB-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.666058] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C41-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.666098] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EB-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.692934] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C689B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.704243] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDAD-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.708777] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224182386870003, replayable_scn:1755224182386870003, readable_scn:1755224182386870003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224183705584, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224182386870003, replayable_scn:1755224182386870003, readable_scn:1755224182386870003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224183705584, ora_rowscn_=1755224183662368000, content_changed=true) [2025-08-15 10:16:23.708967] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:23.708979] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:23.708984] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:23.717284] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8874-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.719919] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8874-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:23.719945] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8874-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.719977] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A53-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.722533] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A53-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:23.722552] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A53-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.727455] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766A-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.730497] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766A-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.732394] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.732481] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766A-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:23.753242] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDBC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.753356] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF086-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.823102] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC651-0-0] [lt=20] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.850948] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D3C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.967134] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B396-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:23.993225] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CD-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:23.994361] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5907-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.995098] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9EA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:23.995828] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CD-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:23.995852] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CD-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.016393] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8899-0-0] [lt=48] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.019029] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8899-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:24.019047] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8899-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.046571] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB60-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.049076] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB60-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:24.049092] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB60-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.075019] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38EB-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:24.077512] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38EB-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:24.078418] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.080951] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839B-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224183889428000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224183889428001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224183889428001, v:0}}], last_dump_ts_=1755224179052005, force_dump=false) [2025-08-15 10:16:24.080976] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839B-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.082984] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.083116] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.106845] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5901-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.139063] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0E2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.141810] INFO [RS] finish (ob_mview_maintenance_task.cpp:280) [438022][T1006_TimerWK1_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=32] mview maintenace task finish(tenant_id_=1006, round_=105668, status_=100, error_code_=0, cost_us_=20001495, prepare_cost_us_=0, gc_mview_cost_us_=1329, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:16:24.151396] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF083-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.188269] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D3A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.198187] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E90D-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:24.200754] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E90D-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:24.202760] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9EF-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.204126] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5282-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:24.206629] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5282-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:24.286484] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D7B-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}) [2025-08-15 10:16:24.286529] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D7B-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=51, ddl_arg=NULL) [2025-08-15 10:16:24.292783] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5321-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:24.292824] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B8-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:24.293964] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5321-0-0] [lt=14] log restore source is empty, just skip(ret=-4018, user_tenant_id=1016) [2025-08-15 10:16:24.295339] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B8-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:24.296412] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5321-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:24.361505] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC7-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:24.364032] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC7-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:24.399537] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62CB-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:24.403146] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62CB-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:24.444119] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6899-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.462742] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:24.462753] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:24.462761] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:24.464115] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:24.471369] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227784470202}) [2025-08-15 10:16:24.471385] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:24.471396] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:24.477226] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227784476071}) [2025-08-15 10:16:24.477240] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:24.477250] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:24.478408] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227784477261}) [2025-08-15 10:16:24.478421] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:24.478428] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:24.503858] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0E0-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.503924] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E74F-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.512859] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF22B-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.527564] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF92-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.529527] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3A-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.530075] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF92-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:24.530092] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF92-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.532021] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3A-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:24.532039] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.534603] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224182567775002, replayable_scn:1755224182567775002, readable_scn:1755224182567775002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224184531250, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224182567775002, replayable_scn:1755224182567775002, readable_scn:1755224182567775002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224184531250, ora_rowscn_=1755224183729654000, content_changed=true) [2025-08-15 10:16:24.534802] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:24.534816] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:24.534828] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:24.535354] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:24.535370] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:24.535376] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:24.536135] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:24.540828] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:24.540838] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:24.540846] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:24.542499] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5842-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82226562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}) [2025-08-15 10:16:24.542544] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5842-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:16:24.546961] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:24.546972] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:24.546980] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:24.546989] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1008, cost_time_us=183, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:24.552265] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:24.552276] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:24.552284] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:24.553568] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:24.612996] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:104) [17458][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=14] start checking lost replicas [2025-08-15 10:16:24.635093] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:24.636326] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224184635018, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:16:24.636351] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224189636348, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:16:24.662612] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.665663] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.666894] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787B-0-0] [lt=10] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224184389147003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224184389147003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224184389147003, v:0}}], last_dump_ts_=1755224179636552, force_dump=false) [2025-08-15 10:16:24.666914] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787B-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.666936] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC074-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.667125] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EC-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.667136] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C42-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224184389147002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224184389147003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224184389147003, v:0}}], last_dump_ts_=1755224179636098, force_dump=false) [2025-08-15 10:16:24.667160] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C42-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.667607] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.667701] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.670143] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787B-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.670167] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C42-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224184389147002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224184389147003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224184389147003, v:0}}], last_dump_ts_=1755224179639159, force_dump=false) [2025-08-15 10:16:24.670189] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C42-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.670349] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC074-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.670367] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EC-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.672130] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C42-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.672241] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C42-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.672585] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787B-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.672703] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787B-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.673067] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC074-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.673066] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EC-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.673173] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC074-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.673282] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EC-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.693015] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C689C-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.704313] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDAE-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.708981] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:24.709001] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:24.709010] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:24.710383] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:24.722706] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8875-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.725301] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8875-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:24.725319] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8875-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.725342] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A54-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.727917] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A54-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:24.727935] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A54-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:24.733454] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766B-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224184570175001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224184570175002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224184570175002, v:0}}], last_dump_ts_=1755224179703403, force_dump=false) [2025-08-15 10:16:24.733478] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766B-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.736392] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766B-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224184390201000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224184390201001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224184390201001, v:0}}], last_dump_ts_=1755224179706375, force_dump=false) [2025-08-15 10:16:24.736411] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766B-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.738262] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766B-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.738363] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:24.753315] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDBD-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.753429] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF087-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.851022] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D3D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.936753] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224184549125, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224182386870002, replayable_scn:1755224182386870002, readable_scn:1755224182386870002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:24.968562] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B397-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:24.994438] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5908-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.995174] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9EB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:24.998551] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CE-0-0] [lt=47] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.001095] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CE-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:25.001113] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CE-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.011977] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:25.021808] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889A-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.024367] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889A-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:25.024384] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.051848] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB61-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.054381] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB61-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:25.054398] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB61-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.082837] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38EC-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:25.084254] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839C-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224184890945000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224184890945001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224184890945001, v:0}}], last_dump_ts_=1755224180055290, force_dump=false) [2025-08-15 10:16:25.084279] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.085374] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38EC-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:25.086677] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839C-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.088597] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.088692] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.106924] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5902-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.139138] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0E3-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.151471] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF084-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.188344] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D3B-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.202837] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9F0-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.206001] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E90E-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:25.208503] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E90E-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:25.211872] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5283-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:25.214320] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5283-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:25.300743] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B9-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:25.301666] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5322-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:25.303482] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86B9-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:25.304119] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5322-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:25.361576] INFO [RS.LB] idle (ob_root_balancer.cpp:87) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=29] root balance idle(idle_time=10000000) [2025-08-15 10:16:25.361593] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:70) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=13] start check unit migrate finish [2025-08-15 10:16:25.361609] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1) [2025-08-15 10:16:25.369371] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC8-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:25.370964] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=6] try check migrate unit finish by tenant(tenant_id=1004) [2025-08-15 10:16:25.371880] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5A-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1011, thread_idx=0) [2025-08-15 10:16:25.371914] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC8-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:25.373202] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1011, only_for_display=false) [2025-08-15 10:16:25.374458] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:25.389730] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.390889] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.392052] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.392075] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1011, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:25.392088] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.392098] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1011, ls_id:{id:1}, replicas:[{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:25.392217] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=92] build ls locality stat map(tenant_id=1011, ls_id={id:1}) [2025-08-15 10:16:25.392225] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.392237] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.392245] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.392254] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:25.392331] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.392344] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.392355] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.392363] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.392370] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3825, tenant_id=1011, ls_id={id:1}) [2025-08-15 10:16:25.392379] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:25.393781] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=20580, tenant_id=1011, acc_dr_task=0) [2025-08-15 10:16:25.395016] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1012, only_for_display=false) [2025-08-15 10:16:25.396217] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:25.396534] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=12] try check migrate unit finish by tenant(tenant_id=1006) [2025-08-15 10:16:25.408524] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62CC-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:25.411025] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62CC-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:25.411355] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.412937] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.414239] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=29] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.414261] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:25.414276] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.414286] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1}, replicas:[{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:25.414393] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=84] build ls locality stat map(tenant_id=1012, ls_id={id:1}) [2025-08-15 10:16:25.414402] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.414414] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.414422] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.414432] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:25.414510] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.414522] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.414529] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.414537] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.414542] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=4343, tenant_id=1012, ls_id={id:1}) [2025-08-15 10:16:25.414550] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:25.416329] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:25.422100] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=16] try check migrate unit finish by tenant(tenant_id=1008) [2025-08-15 10:16:25.438642] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.439789] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.440936] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.440961] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:25.440977] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.440988] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1001}, replicas:[{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:25.441095] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=83] build ls locality stat map(tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:16:25.441103] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.441116] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.441125] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.441134] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:25.441218] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.441230] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.441238] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.441245] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.441252] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3752, tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:16:25.441259] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:25.442800] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:25.444192] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C689A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.447523] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1012) [2025-08-15 10:16:25.462908] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:25.462920] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:25.462927] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:25.464838] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.465981] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.467115] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.467137] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:25.467153] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.467162] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1002}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:25.467271] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=85] build ls locality stat map(tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:16:25.467282] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.467296] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.467307] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.467317] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:25.467395] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.467408] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.467416] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.467424] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.467430] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3730, tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:16:25.467439] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:25.468994] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:25.473046] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1014) [2025-08-15 10:16:25.475391] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D39-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.93164062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}) [2025-08-15 10:16:25.475437] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D39-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=51, ddl_arg=NULL) [2025-08-15 10:16:25.491089] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.492246] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.493392] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.493415] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:25.493428] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.493442] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1003}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:25.493553] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=87] build ls locality stat map(tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:16:25.493562] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.493576] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.493582] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=4] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.493594] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:25.493672] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.493683] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.493692] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.493698] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.493705] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3782, tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:16:25.493714] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:25.495237] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=101441, tenant_id=1012, acc_dr_task=0) [2025-08-15 10:16:25.498695] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1016) [2025-08-15 10:16:25.503936] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0E1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.504003] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E750-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.524210] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1018) [2025-08-15 10:16:25.532824] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF93-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.534704] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3B-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.534812] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:25.534830] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:25.534838] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:25.535379] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF93-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:25.535384] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:25.535395] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:25.535401] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:25.535395] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF93-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.537297] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3B-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:25.537312] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3B-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.540710] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224183388167003, replayable_scn:1755224183388167003, readable_scn:1755224183388167003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224185537705, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224183388167003, replayable_scn:1755224183388167003, readable_scn:1755224183388167003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224185537705, ora_rowscn_=1755224184664757000, content_changed=true) [2025-08-15 10:16:25.540890] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:25.540902] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:25.540907] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:25.547015] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:25.547027] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:25.547033] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:25.549928] INFO [RS] check (ob_alter_locality_finish_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81575-0-0] [lt=14] start to check alter locality finish [2025-08-15 10:16:25.551096] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81576-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.552282] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:25.552285] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81576-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.552292] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:25.552300] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:25.554577] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81578-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.555716] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81578-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.558002] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8157A-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.559155] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8157A-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.561447] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8157C-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.562602] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8157C-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.564898] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8157E-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.566075] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8157E-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.568353] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81580-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.569487] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81580-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.571780] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81582-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.572958] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81582-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.575248] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81584-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.576383] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81584-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.577543] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81584-0-0] [lt=12] start check shrink resource pool [2025-08-15 10:16:25.577560] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81585-0-0] [lt=11] start check shrink resource pool(tenant_id=1) [2025-08-15 10:16:25.578668] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81585-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.579802] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81585-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.579852] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81586-0-0] [lt=11] start check shrink resource pool(tenant_id=1004) [2025-08-15 10:16:25.581002] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81586-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.582138] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81586-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.582190] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81587-0-0] [lt=12] start check shrink resource pool(tenant_id=1006) [2025-08-15 10:16:25.583301] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81587-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.584443] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81587-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.584492] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81588-0-0] [lt=12] start check shrink resource pool(tenant_id=1008) [2025-08-15 10:16:25.585610] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81588-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.586743] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81588-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.586793] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81589-0-0] [lt=12] start check shrink resource pool(tenant_id=1012) [2025-08-15 10:16:25.587897] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81589-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.589036] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81589-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.589084] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158A-0-0] [lt=12] start check shrink resource pool(tenant_id=1014) [2025-08-15 10:16:25.590176] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158A-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.591396] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158A-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.591445] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158B-0-0] [lt=11] start check shrink resource pool(tenant_id=1016) [2025-08-15 10:16:25.592565] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158B-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.593720] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158B-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.593768] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158C-0-0] [lt=11] start check shrink resource pool(tenant_id=1018) [2025-08-15 10:16:25.594864] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158C-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.596013] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158C-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.596063] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:99) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158C-0-0] [lt=12] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.596077] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158C-0-0] [lt=9] start check primary zone [2025-08-15 10:16:25.596089] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158D-0-0] [lt=7] start check primary zone(tenant_id=1) [2025-08-15 10:16:25.597204] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158D-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.597252] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158E-0-0] [lt=11] start check primary zone(tenant_id=1004) [2025-08-15 10:16:25.598357] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158E-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.598404] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158F-0-0] [lt=11] start check primary zone(tenant_id=1006) [2025-08-15 10:16:25.599516] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8158F-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.599565] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81590-0-0] [lt=12] start check primary zone(tenant_id=1008) [2025-08-15 10:16:25.600661] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81590-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.600709] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81591-0-0] [lt=11] start check primary zone(tenant_id=1012) [2025-08-15 10:16:25.601835] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81591-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.601883] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81592-0-0] [lt=11] start check primary zone(tenant_id=1014) [2025-08-15 10:16:25.602996] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81592-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.603042] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81593-0-0] [lt=10] start check primary zone(tenant_id=1016) [2025-08-15 10:16:25.604152] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81593-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.604198] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=10] start check primary zone(tenant_id=1018) [2025-08-15 10:16:25.605309] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:25.605363] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:166) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=13] start do unit balance [2025-08-15 10:16:25.611501] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:350) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=8] start distribute for server status change [2025-08-15 10:16:25.611601] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:401) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=12] finish distribute for server status change(ret=0) [2025-08-15 10:16:25.611612] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:200) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=9] finish do unit balance(ret=0) [2025-08-15 10:16:25.617606] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:115) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=4] start to do tenant group unit balance [2025-08-15 10:16:25.617647] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=12] start to do tenant group unit balance(zone=zone1) [2025-08-15 10:16:25.617663] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=9] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.203.1.34:2882", disk_in_use_:3089104896, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:25.617683] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=14] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:25.617717] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=14] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:25.617737] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=9] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:25.617751] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=10] start do non-tenantgroup unit balance task(zone=zone1) [2025-08-15 10:16:25.617760] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=6] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:25.617784] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=14] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.1.34:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:25.617838] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=42] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.203.1.34:2882", disk_in_use_:3089104896, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:25.617850] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=10] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.1.34:2882", disk_in_use_:3089104896, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:25.617866] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=9] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:25.617879] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=11] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1006, unit_config:{unit_config_id:1005, name:"config_opsone_zone1_s_2_4_ywx", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7988, unit_config:0x7f1e9aa71c38, pool:0x7f1c3beb2f18}, {unit_id:1008, tenant_id:1006, resource_pool_id:1008, unit_config:{unit_config_id:1008, name:"config_test_zone1_s_2_4_emp", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f77d8, unit_config:0x7f1e9aa71998, pool:0x7f1c3beb27a8}, {unit_id:1011, tenant_id:1008, resource_pool_id:1011, unit_config:{unit_config_id:1011, name:"config_mcp_zone1_s_4_8_tmc", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7550, unit_config:0x7f1e9aa716f8, pool:0x7f1e443fda40}, {unit_id:1016, tenant_id:1012, resource_pool_id:1016, unit_config:{unit_config_id:1017, name:"config_oms_zone1_s_4_8_gzp", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f73a0, unit_config:0x7f1e9aa71458, pool:0x7f1e443fd2d0}, {unit_id:1019, tenant_id:1014, resource_pool_id:1019, unit_config:{unit_config_id:1019, name:"config_grafana_test_zone1_s_1_2_pri", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7118, unit_config:0x7f1e9aa71298, pool:0x7f1e443fc7a8}, {unit_id:1024, tenant_id:1016, resource_pool_id:1024, unit_config:{unit_config_id:1023, name:"config_temporalio_zone1_s_2_4_ykw", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6ce0, unit_config:0x7f1e9aa70f18, pool:0x7f1e74dff2d0}, {unit_id:1026, tenant_id:1018, resource_pool_id:1026, unit_config:{unit_config_id:1027, name:"config_media_zone1_s_2_4_kmn", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6b30, unit_config:0x7f1e9aa70b98, pool:0x7f1e74dfeb60}], not_grant_units=[], available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:25.617948] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=34] start to do tenant group unit balance(zone=zone2) [2025-08-15 10:16:25.617959] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=7] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.203.4.38:2882", disk_in_use_:3135242240, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:25.617972] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=9] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:25.617991] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=9] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:25.618003] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=7] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:25.618012] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone2) [2025-08-15 10:16:25.618017] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:25.618031] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.38:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:25.618063] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=25] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.203.4.38:2882", disk_in_use_:3135242240, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:25.618071] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=5] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.38:2882", disk_in_use_:3135242240, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:25.618082] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=9] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:25.618088] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=5] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1004, unit_config:{unit_config_id:1004, name:"config_opsone_zone2_s_2_4_wag", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7b38, unit_config:0x7f1e9aa71d18, pool:0x7f1c3beb3688}, {unit_id:1009, tenant_id:1006, resource_pool_id:1009, unit_config:{unit_config_id:1009, name:"config_test_zone2_s_2_4_yey", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7700, unit_config:0x7f1e9aa718b8, pool:0x7f1c3beb23f0}, {unit_id:1010, tenant_id:1008, resource_pool_id:1010, unit_config:{unit_config_id:1010, name:"config_mcp_zone2_s_4_8_bjb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7628, unit_config:0x7f1e9aa717d8, pool:0x7f1c3beb2038}, {unit_id:1018, tenant_id:1012, resource_pool_id:1018, unit_config:{unit_config_id:1016, name:"config_oms_zone2_s_4_8_kbx", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f71f0, unit_config:0x7f1e9aa71538, pool:0x7f1e443fcb60}, {unit_id:1021, tenant_id:1014, resource_pool_id:1021, unit_config:{unit_config_id:1021, name:"config_grafana_test_zone2_s_1_2_yyc", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f6f68, unit_config:0x7f1e9aa710d8, pool:0x7f1e443fc038}, {unit_id:1023, tenant_id:1016, resource_pool_id:1023, unit_config:{unit_config_id:1022, name:"config_temporalio_zone2_s_2_4_lsv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6db8, unit_config:0x7f1e9aa70ff8, pool:0x7f1e74dff688}, {unit_id:1027, tenant_id:1018, resource_pool_id:1027, unit_config:{unit_config_id:1025, name:"config_media_zone2_s_2_4_yzd", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6a58, unit_config:0x7f1e9aa70d58, pool:0x7f1e74dfe7a8}], not_grant_units=[], available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:25.618140] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=30] start to do tenant group unit balance(zone=zone3) [2025-08-15 10:16:25.618149] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.203.4.39:2882", disk_in_use_:3034578944, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:25.618162] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=9] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:25.618174] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=6] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:25.618186] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=7] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:25.618195] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone3) [2025-08-15 10:16:25.618200] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:25.618213] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.39:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:25.618244] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=24] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.203.4.39:2882", disk_in_use_:3034578944, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:25.618258] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=10] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.39:2882", disk_in_use_:3034578944, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:25.618269] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:25.618275] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=5] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1005, unit_config:{unit_config_id:1006, name:"config_opsone_zone3_s_2_4_vxv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7a60, unit_config:0x7f1e9aa71b58, pool:0x7f1c3beb32d0}, {unit_id:1007, tenant_id:1006, resource_pool_id:1007, unit_config:{unit_config_id:1007, name:"config_test_zone3_s_2_4_mui", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f78b0, unit_config:0x7f1e9aa71a78, pool:0x7f1c3beb2b60}, {unit_id:1012, tenant_id:1008, resource_pool_id:1012, unit_config:{unit_config_id:1012, name:"config_mcp_zone3_s_4_8_seu", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7478, unit_config:0x7f1e9aa71618, pool:0x7f1e443fd688}, {unit_id:1017, tenant_id:1012, resource_pool_id:1017, unit_config:{unit_config_id:1018, name:"config_oms_zone3_s_4_8_nvb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f72c8, unit_config:0x7f1e9aa71378, pool:0x7f1e443fcf18}, {unit_id:1020, tenant_id:1014, resource_pool_id:1020, unit_config:{unit_config_id:1020, name:"config_grafana_test_zone3_s_1_2_xbz", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7040, unit_config:0x7f1e9aa711b8, pool:0x7f1e443fc3f0}, {unit_id:1022, tenant_id:1016, resource_pool_id:1022, unit_config:{unit_config_id:1024, name:"config_temporalio_zone3_s_2_4_rxz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6e90, unit_config:0x7f1e9aa70e38, pool:0x7f1e74dffa40}, {unit_id:1025, tenant_id:1018, resource_pool_id:1025, unit_config:{unit_config_id:1026, name:"config_media_zone3_s_2_4_ukz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6c08, unit_config:0x7f1e9aa70c78, pool:0x7f1e74dfef18}], not_grant_units=[], available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:25.630157] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:25.631421] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224185630082, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:16:25.631445] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224190631443, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:16:25.668704] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.671751] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.673236] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C43-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.673621] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.673656] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED8-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1007, thread_idx=0) [2025-08-15 10:16:25.673699] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.673714] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.674117] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC075-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.674240] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413ED-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.674929] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1007, only_for_display=false) [2025-08-15 10:16:25.676179] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=19] start try ls disaster recovery(ls_status_info={tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:25.676377] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C43-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.676626] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787C-0-0] [lt=23] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.677080] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC075-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.677285] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413ED-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.678375] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C43-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.678478] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C43-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.678632] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.678718] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787C-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.679005] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC075-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.679119] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC075-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.679255] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413ED-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.679336] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413ED-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.690249] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.691441] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.692630] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.692653] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1007, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:25.692667] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.692679] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1007, ls_id:{id:1}, replicas:[{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:25.692784] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=82] build ls locality stat map(tenant_id=1007, ls_id={id:1}) [2025-08-15 10:16:25.692792] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.692808] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.692820] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.692832] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:25.692916] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=63] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.692933] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.692948] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=11] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.692953] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.692961] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3888, tenant_id=1007, ls_id={id:1}) [2025-08-15 10:16:25.692972] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:25.693092] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C689D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.694076] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19147, tenant_id=1007, acc_dr_task=0) [2025-08-15 10:16:25.695277] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1008, only_for_display=false) [2025-08-15 10:16:25.696468] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:25.704390] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDAF-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.709024] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:25.709037] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:25.709043] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:25.710309] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.711477] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.712644] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.712668] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:25.712687] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.712699] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1}, replicas:[{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:25.712801] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=80] build ls locality stat map(tenant_id=1008, ls_id={id:1}) [2025-08-15 10:16:25.712809] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.712823] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.712830] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.712842] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:25.712919] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.712933] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.712949] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=12] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.712954] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.712962] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3819, tenant_id=1008, ls_id={id:1}) [2025-08-15 10:16:25.712972] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:25.714035] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:25.728035] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8876-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.730602] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8876-0-0] [lt=68] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:25.730621] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8876-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.730697] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A55-0-0] [lt=37] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.733312] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A55-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:25.733331] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A55-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:25.734444] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [3414234][T1006_TimerWK4_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=8] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:25.734936] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.736093] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.737255] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.737277] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:25.737291] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.737300] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1001}, replicas:[{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:25.737406] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=82] build ls locality stat map(tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:16:25.737414] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.737425] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.737433] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.737441] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:25.737517] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.737530] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.737537] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.737544] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.737548] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3744, tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:16:25.737556] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:25.738636] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:25.739361] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.742342] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766C-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.744204] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.744301] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:25.753385] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDBE-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.753503] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF088-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.759518] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.760694] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.761878] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.761899] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:25.761912] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.761922] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1002}, replicas:[{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:25.762027] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=82] build ls locality stat map(tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:16:25.762034] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.762045] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.762054] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.762063] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:25.762157] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=70] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.762169] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.762176] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.762183] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.762189] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3866, tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:16:25.762197] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:25.763324] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:25.784143] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.785282] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.786426] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:25.786450] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:25.786466] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.786477] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1003}, replicas:[{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:25.786580] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=81] build ls locality stat map(tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:16:25.786592] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.786607] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.786613] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:25.786623] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:25.786701] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.786711] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.786720] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.786725] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:25.786735] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3726, tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:16:25.786745] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:25.787794] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93703, tenant_id=1008, acc_dr_task=0) [2025-08-15 10:16:25.828023] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC652-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.851097] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D3E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.954775] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:25.956027] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224185954711, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:16:25.956051] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224190956049, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:16:25.970031] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B398-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:25.994510] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5909-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:25.995251] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9EC-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.004433] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CF-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.006970] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CF-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:26.006988] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95CF-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.027084] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889B-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.029642] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889B-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:26.029660] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.057202] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB62-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.059737] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB62-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:26.059755] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB62-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.077394] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:26.078647] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224186068605, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:16:26.078673] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224191078671, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:16:26.078690] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224191078687, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:26.089706] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.091795] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38ED-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:26.092294] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.094284] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.094384] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38ED-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:26.094463] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.107012] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5903-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.139209] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0E4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.151542] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF085-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.188414] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D3C-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.197606] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:26.198871] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224186197526, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:16:26.198894] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224191198893, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:16:26.199743] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:26.200943] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224186199685, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:16:26.200964] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224191200963, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:26.202911] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9F1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.213876] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E90F-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:26.216371] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E90F-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:26.219607] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5284-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:26.222148] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5284-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:26.286992] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D7C-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}) [2025-08-15 10:16:26.287033] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D7C-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:16:26.308988] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86BA-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:26.309430] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5323-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:26.311513] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86BA-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:26.311885] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5323-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:26.372460] INFO [RS] finish (ob_mview_maintenance_task.cpp:280) [727959][T1018_TimerWK3_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=10] mview maintenace task finish(tenant_id_=1018, round_=63408, status_=100, error_code_=0, cost_us_=20001392, prepare_cost_us_=0, gc_mview_cost_us_=1231, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:16:26.377394] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC9-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:26.379882] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FC9-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:26.416361] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62CD-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:26.418869] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62CD-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:26.444269] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C689B-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.462670] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224184889890003, replayable_scn:1755224184889890003, readable_scn:1755224184889890003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224186459307, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224184889890003, replayable_scn:1755224184889890003, readable_scn:1755224184889890003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224186459307, ora_rowscn_=1755224186091104000, content_changed=true) [2025-08-15 10:16:26.462855] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:26.462874] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:26.462886] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:26.504003] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0E2-0-0] [lt=24] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.504074] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E751-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.513143] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF22C-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.534841] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:26.534853] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:26.534859] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:26.535330] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224184390201002, replayable_scn:1755224184390201002, readable_scn:1755224184390201002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224186531882, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224184390201002, replayable_scn:1755224184390201002, readable_scn:1755224184390201002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224186531882, ora_rowscn_=1755224185675455000, content_changed=true) [2025-08-15 10:16:26.535507] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:26.535524] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:26.535532] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:26.538156] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF94-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.539994] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3C-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.540732] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF94-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:26.540751] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF94-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.540925] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:26.540952] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:26.540965] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:26.542345] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:26.542480] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3C-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:26.542499] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.542987] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBABA-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83007812GB}) [2025-08-15 10:16:26.543031] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBABA-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:16:26.546910] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224184389147003, replayable_scn:1755224184389147003, readable_scn:1755224184389147003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224186543480, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224184389147003, replayable_scn:1755224184389147003, readable_scn:1755224184389147003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224186543480, ora_rowscn_=1755224185675455000, content_changed=true) [2025-08-15 10:16:26.547077] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:26.547087] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:26.547095] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:26.552195] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224184389147003, replayable_scn:1755224184389147003, readable_scn:1755224184389147003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224186548745, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224184389147003, replayable_scn:1755224184389147003, readable_scn:1755224184389147003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224186548745, ora_rowscn_=1755224185675455000, content_changed=true) [2025-08-15 10:16:26.552370] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:26.552380] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:26.552387] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:26.572788] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487710][T1003_DRService][T1003][YB420ACB0122-0006337FE6577D86-0-0] [lt=15] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1003, thread_idx=1) [2025-08-15 10:16:26.576413] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [1487710][T1003_DRService][T1003][YB420ACB0122-0006337FE6577D87-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2336, tenant_id=1003, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:26.578708] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [1487710][T1003_DRService][T1003][YB420ACB0122-0006337FE6577D87-0-0] [lt=13] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2278, tenant_id=1004, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:26.674648] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770F-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.677845] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770F-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.679420] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C44-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.679644] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787D-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.679771] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.679860] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D770F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.680048] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC076-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.680257] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EE-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224186391931003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224186391931003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224186391931003, v:0}}], last_dump_ts_=1755224181648874, force_dump=false) [2025-08-15 10:16:26.680283] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EE-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.682525] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C44-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.682643] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787D-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.683121] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC076-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224186391931002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224186665432000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224186665432000, v:0}}], last_dump_ts_=1755224181651590, force_dump=false) [2025-08-15 10:16:26.683143] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC076-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.683257] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EE-0-0] [lt=15] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224186391931003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224186391931003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224186391931003, v:0}}], last_dump_ts_=1755224181651734, force_dump=false) [2025-08-15 10:16:26.683279] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EE-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.684504] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C44-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.684590] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.684603] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C44-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.684665] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.685101] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC076-0-0] [lt=3] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.685192] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC076-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.685291] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EE-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.685375] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EE-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.693164] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C689E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.704470] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDB0-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.708930] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224185391452002, replayable_scn:1755224185391452002, readable_scn:1755224185391452002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224186705320, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224185391452002, replayable_scn:1755224185391452002, readable_scn:1755224185391452002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224186705320, ora_rowscn_=1755224186681167000, content_changed=true) [2025-08-15 10:16:26.709103] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:26.709114] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:26.709121] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:26.717148] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAB2-0-0] [lt=11] [RESTORE] restore scheduler quit [2025-08-15 10:16:26.718396] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAB2-0-0] [lt=11] get targets(ret=0, jobs=[], exec_tenant_id=1017, sql=select * from __all_recover_table_job) [2025-08-15 10:16:26.719583] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAB3-0-0] [lt=13] get targets(ret=0, jobs=[], exec_tenant_id=1017, sql=select * from __all_import_table_job) [2025-08-15 10:16:26.719604] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAB4-0-0] [lt=12][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224186719599, ref_timestamp=1755224126715262, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:16:26.719685] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAB5-0-0] [lt=9] [RESTORE] restore scheduler start [2025-08-15 10:16:26.719693] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAB6-0-0] [lt=6] [RESTORE] try process restore job [2025-08-15 10:16:26.720831] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAB6-0-0] [lt=5] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:16:26.733304] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8877-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.735844] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8877-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:26.735861] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8877-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.736499] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A56-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.739017] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A56-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:26.739032] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A56-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:26.745394] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.748423] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.750477] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.750567] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:26.753457] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDBF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.753574] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF089-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.789050] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=16] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:26.838979] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=6] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:26.840199] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224186838908, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:16:26.840225] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224191840223, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:16:26.845021] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475908][T1008_TimerWK0_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=16] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:26.851171] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D3F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.971520] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B399-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:26.994585] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C590A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:26.995323] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9ED-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.009707] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D0-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.012311] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D0-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:27.012330] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D0-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.032431] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889C-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.034971] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889C-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:27.034989] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.062553] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB63-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.065087] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB63-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:27.065099] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB63-0-0] [lt=10] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.095472] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.098646] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.099833] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38EE-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:27.100758] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.100853] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839E-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.102681] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38EE-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:27.107090] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5904-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.139279] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0E5-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.151620] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF086-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.179337] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727135][T1016_TimerWK3_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:27.188489] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D3D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.202947] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9F2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.214634] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727959][T1018_TimerWK3_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=10] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:27.221712] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E910-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:27.224203] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E910-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:27.227461] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5285-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:27.229964] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5285-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:27.268986] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619487][T1012_TimerWK2_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=29] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:27.296182] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [17709][T1_REST_SER][T1][YB420ACB0122-0006337F03BA32BB-0-0] [lt=14] [RESTORE] restore scheduler quit [2025-08-15 10:16:27.297454] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [17709][T1_REST_SER][T1][YB420ACB0122-0006337F03BA32BB-0-0] [lt=16] get targets(ret=0, jobs=[], exec_tenant_id=1, sql=select * from __all_recover_table_job) [2025-08-15 10:16:27.297481] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [17709][T1_REST_SER][T1][YB420ACB0122-0006337F03BA32BC-0-0] [lt=15][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224187297475, ref_timestamp=1755224127294689, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:16:27.297565] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [17709][T1_REST_SER][T1][YB420ACB0122-0006337F03BA32BD-0-0] [lt=11] [RESTORE] restore scheduler start [2025-08-15 10:16:27.297573] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [17709][T1_REST_SER][T1][YB420ACB0122-0006337F03BA32BE-0-0] [lt=6] [RESTORE] try process restore job [2025-08-15 10:16:27.298742] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [17709][T1_REST_SER][T1][YB420ACB0122-0006337F03BA32BE-0-0] [lt=3] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:16:27.316806] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86BB-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:27.317125] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5324-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:27.319621] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5324-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:27.320447] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86BB-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:27.329364] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487604][T1004_TimerWK1_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=47] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:16:27.343893] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:27.345139] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224187343820, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:16:27.345163] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224192345161, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:16:27.385204] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FCA-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:27.387706] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FCA-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:27.424208] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62CE-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:27.426685] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62CE-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:27.435481] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:27.436724] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224187435406, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:16:27.436752] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224192436749, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:16:27.444344] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C689C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.447373] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E64-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224187446422000, start_id=1755224187446422000, end_id=1755224187446422001) [2025-08-15 10:16:27.448044] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E64-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224187446422000, v:0}, tenant_id=1017) [2025-08-15 10:16:27.462893] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:27.462911] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:27.462917] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:27.475878] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A0916-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}) [2025-08-15 10:16:27.475919] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A0916-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=47, ddl_arg=NULL) [2025-08-15 10:16:27.482700] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227787481534}) [2025-08-15 10:16:27.482717] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:27.482728] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:27.488664] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227787487491}) [2025-08-15 10:16:27.488678] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:27.488689] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:27.489861] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227787488700}) [2025-08-15 10:16:27.489874] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:27.489885] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:27.504077] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0E3-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.504146] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E752-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.534729] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224185571359003, replayable_scn:1755224185571359003, readable_scn:1755224185571359003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224187531489, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224185571359003, replayable_scn:1755224185571359003, readable_scn:1755224185571359003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224187531489, ora_rowscn_=1755224186747383000, content_changed=true) [2025-08-15 10:16:27.534910] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:27.534922] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:27.534928] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:27.535549] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:27.535561] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:27.535569] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:27.536855] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:27.540946] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:27.540958] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:27.540964] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:27.543477] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF95-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.545179] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3D-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.546062] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF95-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:27.546079] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF95-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.547133] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:27.547145] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:27.547151] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:27.547158] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1008, cost_time_us=176, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:27.547700] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3D-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:27.547717] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3D-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.548446] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:27.552388] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:27.552400] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:27.552406] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:27.581991] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E6D-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224187581910000, start_id=1755224187581910000, end_id=1755224187581910001) [2025-08-15 10:16:27.582662] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E6D-0-0] [lt=13] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224187581910000, v:0}, tenant_id=1015) [2025-08-15 10:16:27.582947] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66F-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224187581910000, start_id=1755224187581910000, end_id=1755224187581910001) [2025-08-15 10:16:27.583617] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66F-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224187581910000, v:0}, tenant_id=1005) [2025-08-15 10:16:27.680889] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7710-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.684016] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7710-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.685673] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C45-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.685686] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.686209] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7710-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.686196] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC077-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224187393880001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224187393880002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224187393880002, v:0}}], last_dump_ts_=1755224182654723, force_dump=false) [2025-08-15 10:16:27.686228] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC077-0-0] [lt=24] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.686293] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7710-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.686328] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EF-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.688616] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C45-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.688775] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787E-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224187393880003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224187393880002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224187393880003, v:0}}], last_dump_ts_=1755224182657807, force_dump=false) [2025-08-15 10:16:27.688796] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787E-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.689330] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EF-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.689737] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC077-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.690538] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C45-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.690632] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C45-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.690766] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787E-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.690880] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.691374] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EF-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.691468] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413EF-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.691906] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC077-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.692044] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC077-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.693240] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C689F-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.704505] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDB1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.709214] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:27.709230] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:27.709238] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:27.738583] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8878-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.741145] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8878-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:27.741162] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8878-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.741684] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A57-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.744182] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A57-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:27.744199] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A57-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:27.751604] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.753532] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDC0-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.753648] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF08A-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.754636] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.756696] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.756783] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:27.808819] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:27.810079] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224187808738, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:16:27.810103] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224192810101, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:16:27.811262] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224187809948, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:16:27.811282] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224192811281, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:16:27.811328] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36CD-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224187811175000, start_id=1755224187811175000, end_id=1755224187811175001) [2025-08-15 10:16:27.811971] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36CD-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224187811175000, v:0}, tenant_id=1011) [2025-08-15 10:16:27.832909] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC653-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.851245] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D40-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.856117] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=5] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:27.856152] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=23] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:27.856168] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=12] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:27.858026] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [1487606][T1004_TimerWK3_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=16] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:27.858059] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [1487606][T1004_TimerWK3_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=22] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:27.858072] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [1487606][T1004_TimerWK3_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=10] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:27.859216] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=10] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:27.859295] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [1487604][T1004_TimerWK1_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=8] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:27.929717] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB5-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224187928973000, start_id=1755224187928973000, end_id=1755224187928973001) [2025-08-15 10:16:27.930404] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB5-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224187928973000, v:0}, tenant_id=1013) [2025-08-15 10:16:27.938371] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=12] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224187549333, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224185391452002, replayable_scn:1755224185391452002, readable_scn:1755224185391452002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:27.972117] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:27.972906] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B39A-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:27.973390] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224187972039, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:16:27.973416] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224192973414, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:16:27.994659] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C590B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:27.995397] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9EE-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.015031] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D1-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.017560] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D1-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:28.017577] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D1-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.037689] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889D-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.040252] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889D-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:28.040271] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.057738] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:28.058994] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224188057667, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:16:28.059018] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224193059016, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:16:28.067878] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB64-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.070400] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB64-0-0] [lt=68] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:28.070417] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB64-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.101874] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.104401] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839F-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.106350] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.106433] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D839F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.107168] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5905-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.108246] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38EF-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:28.109756] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B5-0-0] [lt=71] get number(ret=0, service_type_=0, range=1, base_id=1755224188109634000, start_id=1755224188109634000, end_id=1755224188109634001) [2025-08-15 10:16:28.110536] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B5-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224188109634000, v:0}, tenant_id=1012) [2025-08-15 10:16:28.110783] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38EF-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:28.129285] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463BD-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224188128586000, start_id=1755224188128586000, end_id=1755224188128586001) [2025-08-15 10:16:28.130054] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463BD-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224188128586000, v:0}, tenant_id=1) [2025-08-15 10:16:28.139352] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0E6-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.151698] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF087-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.188557] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D3E-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.203021] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9F3-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.229571] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E911-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:28.233334] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E911-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:28.237043] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5286-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:28.239592] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5286-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:28.287492] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B638F-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}) [2025-08-15 10:16:28.287549] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B638F-0-0] [lt=34] [DDL] execute ddl like stmt(ret=0, cost=63, ddl_arg=NULL) [2025-08-15 10:16:28.324965] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5325-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:28.325711] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86BC-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:28.327442] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5325-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:28.328199] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86BC-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:28.392993] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FCB-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:28.395521] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FCB-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:28.432022] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62CF-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:28.434513] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62CF-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:28.444409] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C689D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.462932] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:28.462951] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:28.462957] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:28.504153] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0E4-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.504215] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E753-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.513433] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF22D-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.534946] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:28.534964] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:28.534971] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:28.535605] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:28.535616] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:28.535626] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:28.540829] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224186392986001, replayable_scn:1755224186392986001, readable_scn:1755224186392986001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224188538786, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224186392986001, replayable_scn:1755224186392986001, readable_scn:1755224186392986001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224188538786, ora_rowscn_=1755224187682904000, content_changed=true) [2025-08-15 10:16:28.541005] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:28.541015] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:28.541023] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:28.543490] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAC1-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83007812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83398438GB}) [2025-08-15 10:16:28.543533] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAC1-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:16:28.547172] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:28.547183] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:28.547189] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:28.548831] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF96-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.550352] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3E-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.551404] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF96-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:28.551422] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF96-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.552487] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:28.552499] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:28.552505] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:28.552878] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3E-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:28.552895] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.615896] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:28.617138] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224188615820, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:16:28.617163] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224193617161, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:16:28.687311] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7711-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224188395729001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224188395729001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224188395729001, v:0}}], last_dump_ts_=1755224183656440, force_dump=false) [2025-08-15 10:16:28.687335] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7711-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.690379] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7711-0-0] [lt=7] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224188395729001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224188395729001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224188395729001, v:0}}], last_dump_ts_=1755224183659580, force_dump=false) [2025-08-15 10:16:28.690404] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7711-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.691594] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C46-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.691875] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787F-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.692428] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F0-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.692485] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7711-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.692576] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7711-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.693009] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC078-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.693179] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF3-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1, thread_idx=0) [2025-08-15 10:16:28.693317] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68A0-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.694540] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-08-15 10:16:28.694594] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C46-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.694818] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787F-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.695398] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F0-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.695858] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:28.696015] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC078-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.696555] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C46-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.696636] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C46-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.696874] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787F-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.696968] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB787F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.697330] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F0-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.697424] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F0-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.697867] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8274-0-0] [lt=18] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1005, thread_idx=0) [2025-08-15 10:16:28.698024] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC078-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.698133] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC078-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.699126] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1005, only_for_display=false) [2025-08-15 10:16:28.700325] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:28.704584] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDB2-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.709356] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:28.709376] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:28.709383] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:28.709805] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.710952] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.712098] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.712121] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:28.712138] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.712150] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:28.712253] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=81] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-08-15 10:16:28.712267] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.712281] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.712291] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.712303] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:28.712381] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.712400] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=10] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.712410] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.712416] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.712428] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=9] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3785, tenant_id=1, ls_id={id:1}) [2025-08-15 10:16:28.712441] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=10] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:28.713546] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=10] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19006, tenant_id=1, acc_dr_task=0) [2025-08-15 10:16:28.714138] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.715300] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.716450] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.716472] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1005, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:28.716489] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.716503] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1005, ls_id:{id:1}, replicas:[{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:28.716601] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=77] build ls locality stat map(tenant_id=1005, ls_id={id:1}) [2025-08-15 10:16:28.716608] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.716621] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.716633] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.716644] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:28.716720] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.716735] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.716744] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.716750] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.716759] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3768, tenant_id=1005, ls_id={id:1}) [2025-08-15 10:16:28.716769] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:28.717863] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18737, tenant_id=1005, acc_dr_task=0) [2025-08-15 10:16:28.719090] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1006, only_for_display=false) [2025-08-15 10:16:28.720279] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:28.733983] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.735121] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.736273] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.736296] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:28.736314] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.736328] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1}, replicas:[{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:28.736431] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=80] build ls locality stat map(tenant_id=1006, ls_id={id:1}) [2025-08-15 10:16:28.736439] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.736452] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.736463] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.736475] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:28.736550] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.736565] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.736575] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.736586] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.736592] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3746, tenant_id=1006, ls_id={id:1}) [2025-08-15 10:16:28.736603] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:28.737666] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:28.743852] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8879-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.746429] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8879-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:28.746447] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8879-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.746907] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A58-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.749483] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A58-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:28.749502] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A58-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:28.753606] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDC1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.753721] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF08B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.757750] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766F-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.758265] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.759410] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.760573] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.760596] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:28.760611] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.760618] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=5] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1001}, replicas:[{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:28.760729] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=86] build ls locality stat map(tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:16:28.760725] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766F-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.760737] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.760751] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.760762] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.760773] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:28.760851] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.760865] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.760875] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.760882] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.760892] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3763, tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:16:28.760902] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:28.761980] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:28.762639] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766F-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.762727] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D766F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:28.765253] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5D-0-0] [lt=59] get number(ret=0, service_type_=0, range=1, base_id=1755224188765063000, start_id=1755224188765063000, end_id=1755224188765063001) [2025-08-15 10:16:28.765354] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=15] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:28.765390] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=25] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:28.765404] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=10] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:28.765886] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5D-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224188765063000, v:0}, tenant_id=1016) [2025-08-15 10:16:28.766625] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [727956][T1018_TimerWK0_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=12] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:28.782762] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.783923] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.785064] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.785087] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:28.785100] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.785112] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1002}, replicas:[{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:28.785216] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=81] build ls locality stat map(tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:16:28.785223] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.785237] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.785244] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.785256] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:28.785333] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.785347] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.785358] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.785363] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.785371] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3752, tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:16:28.785379] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:28.786454] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:28.807193] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.808345] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.809485] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:28.809509] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:28.809521] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.809532] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1003}, replicas:[{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:28.809641] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=85] build ls locality stat map(tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:16:28.809654] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.809668] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.809678] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:28.809690] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:28.809770] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=59] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.809784] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.809794] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.809804] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:28.809814] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3786, tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:16:28.809825] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:28.810894] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93013, tenant_id=1006, acc_dr_task=0) [2025-08-15 10:16:28.851319] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D41-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.974312] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B39B-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:28.994734] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C590C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:28.995470] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9EF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.020332] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D2-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.022847] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D2-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:29.022865] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D2-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.042908] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889E-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.045462] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889E-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:29.045477] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889E-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.073167] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB65-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.075704] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB65-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:29.075722] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB65-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.107254] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5906-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.107482] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A0-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.109985] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A0-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224188896355001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224188896355001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224188896355001, v:0}}], last_dump_ts_=1755224184080951, force_dump=false) [2025-08-15 10:16:29.110009] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A0-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.111875] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A0-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.112018] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A0-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.116109] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F0-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:29.118624] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F0-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:29.139423] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0E7-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.151777] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF088-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.181452] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA650-0-0] [lt=11] get number(ret=0, service_type_=0, range=1, base_id=1755224189181203000, start_id=1755224189181203000, end_id=1755224189181203001) [2025-08-15 10:16:29.182194] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA650-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224189181203000, v:0}, tenant_id=1006) [2025-08-15 10:16:29.188593] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D3F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.203093] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9F4-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.238625] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E912-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:29.241154] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E912-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:29.244792] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5287-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:29.245911] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A9-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224189245424000, start_id=1755224189245424000, end_id=1755224189245424001) [2025-08-15 10:16:29.246804] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A9-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224189245424000, v:0}, tenant_id=1008) [2025-08-15 10:16:29.247273] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5287-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:29.332767] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5326-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:29.333577] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86BD-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:29.335272] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5326-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:29.336049] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86BD-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:29.400855] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FCC-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:29.403376] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FCC-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:29.417179] INFO [RS] finish (ob_mview_maintenance_task.cpp:280) [17740][T1_TimerWK3_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=15] mview maintenace task finish(tenant_id_=1, round_=121920, status_=100, error_code_=0, cost_us_=20001381, prepare_cost_us_=0, gc_mview_cost_us_=1213, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:16:29.439775] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D0-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:29.442300] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D0-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:29.444484] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C689E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.457753] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [727356][T1016_MergeSche][T1016][YB420ACB0122-000633A34FF2D66C-0-0] [lt=68] zone_merge_mgr is already loaded(tenant_id=1016, global_merge_info={tenant_id:1016, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402661472, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194765625452, need_update:false}}, zone_merge_infos=[{tenant_id:1016, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194765627764, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402681290, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1016, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194765631279, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402686302, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1016, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194765633133, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402690030, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402541212000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:16:29.462818] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224187894284002, replayable_scn:1755224187894284002, readable_scn:1755224187894284002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224189459957, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224187894284002, replayable_scn:1755224187894284002, readable_scn:1755224187894284002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224189459957, ora_rowscn_=1755224189108621001, content_changed=true) [2025-08-15 10:16:29.463011] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:29.463028] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:29.463036] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:29.464403] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:29.476387] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D3C-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}) [2025-08-15 10:16:29.476432] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D3C-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=50, ddl_arg=NULL) [2025-08-15 10:16:29.504228] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0E5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.504286] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E754-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.535013] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:29.535024] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:29.535029] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:29.535500] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224187393880002, replayable_scn:1755224187393880002, readable_scn:1755224187393880002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224189532535, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224187393880002, replayable_scn:1755224187393880002, readable_scn:1755224187393880002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224189532535, ora_rowscn_=1755224188694510000, content_changed=true) [2025-08-15 10:16:29.535666] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:29.535678] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:29.535686] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:29.536301] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:29.541106] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:29.541123] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:29.541128] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:29.547054] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224187393880002, replayable_scn:1755224187393880002, readable_scn:1755224187393880002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224189544076, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224187393880002, replayable_scn:1755224187393880002, readable_scn:1755224187393880002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224189544076, ora_rowscn_=1755224188693456000, content_changed=true) [2025-08-15 10:16:29.547228] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:29.547238] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:29.547246] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:29.552396] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224187393880002, replayable_scn:1755224187393880002, readable_scn:1755224187393880002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224189549340, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224187393880002, replayable_scn:1755224187393880002, readable_scn:1755224187393880002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224189549340, ora_rowscn_=1755224188694510000, content_changed=true) [2025-08-15 10:16:29.552562] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:29.552571] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:29.552579] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:29.553864] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:29.554131] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF97-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.555505] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3F-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.556674] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF97-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:29.556692] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF97-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.558057] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3F-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:29.558076] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A3F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.636427] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:29.637658] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224189636348, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:16:29.637681] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224194637679, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:16:29.670495] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E66-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1003, thread_idx=0) [2025-08-15 10:16:29.671739] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1003, only_for_display=false) [2025-08-15 10:16:29.672991] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:29.686833] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.688003] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.689166] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.689189] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1003, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:29.689206] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.689218] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1003, ls_id:{id:1}, replicas:[{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:29.689322] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=82] build ls locality stat map(tenant_id=1003, ls_id={id:1}) [2025-08-15 10:16:29.689331] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.689343] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.689354] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.689365] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:29.689441] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.689455] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.689464] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.689471] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.689480] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3801, tenant_id=1003, ls_id={id:1}) [2025-08-15 10:16:29.689489] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:29.690554] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18816, tenant_id=1003, acc_dr_task=0) [2025-08-15 10:16:29.691759] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1004, only_for_display=false) [2025-08-15 10:16:29.692980] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:29.693398] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68A1-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.693622] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7712-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.697101] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7712-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.697609] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C47-0-0] [lt=5] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224189396868001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224189396868002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224189396868002, v:0}}], last_dump_ts_=1755224184667135, force_dump=false) [2025-08-15 10:16:29.697633] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C47-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.698020] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7880-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224189396868003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224189396868003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224189396868003, v:0}}], last_dump_ts_=1755224184666894, force_dump=false) [2025-08-15 10:16:29.698047] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7880-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.698386] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F1-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.699091] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC079-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.699322] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7712-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.699416] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7712-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.700599] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C47-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224189396868001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224189396868002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224189396868002, v:0}}], last_dump_ts_=1755224184670167, force_dump=false) [2025-08-15 10:16:29.700627] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C47-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.701153] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7880-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.701212] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F1-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.701912] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC079-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.702555] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C47-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.702762] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C47-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.703163] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7880-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.703205] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F1-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.703241] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7880-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.703288] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F1-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.703820] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC079-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.703911] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC079-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.704664] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDB3-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.707035] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.708224] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.709093] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224188394677002, replayable_scn:1755224188394677002, readable_scn:1755224188394677002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224189705986, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224188394677002, replayable_scn:1755224188394677002, readable_scn:1755224188394677002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224189705986, ora_rowscn_=1755224189699661000, content_changed=true) [2025-08-15 10:16:29.709258] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:29.709268] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:29.709275] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:29.709395] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.709416] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:29.709428] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.709438] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:29.709548] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=86] build ls locality stat map(tenant_id=1004, ls_id={id:1}) [2025-08-15 10:16:29.709558] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.709569] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.709578] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.709586] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:29.709672] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.709685] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.709693] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.709701] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.709707] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3834, tenant_id=1004, ls_id={id:1}) [2025-08-15 10:16:29.709715] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:29.710544] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:29.710817] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:29.731768] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.732934] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.734125] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=28] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.734153] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:29.734168] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.734175] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=4] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:29.734288] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=88] build ls locality stat map(tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:16:29.734298] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.734310] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.734318] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.734326] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:29.734407] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.734419] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.734427] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.734434] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.734441] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3824, tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:16:29.734449] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:29.735518] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:29.739355] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E9-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224189738511000, start_id=1755224189738511000, end_id=1755224189738511001) [2025-08-15 10:16:29.740059] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E9-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224189738511000, v:0}, tenant_id=1003) [2025-08-15 10:16:29.749178] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887A-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.751714] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887A-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:29.751731] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887A-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.752152] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A59-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.753678] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDC2-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.753798] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF08C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.754636] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A59-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:29.754653] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A59-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:29.756435] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.757618] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.758791] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.758813] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:29.758828] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.758838] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1002}, replicas:[{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:29.758949] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=87] build ls locality stat map(tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:16:29.758958] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.758969] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.758977] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.758986] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:29.759061] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.759073] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.759081] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.759088] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.759094] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3818, tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:16:29.759102] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:29.760207] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:29.763689] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7670-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224189557714001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224189557714002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224189557714002, v:0}}], last_dump_ts_=1755224184733453, force_dump=false) [2025-08-15 10:16:29.763712] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7670-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.766755] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7670-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224189557714001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224189557714002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224189557714002, v:0}}], last_dump_ts_=1755224184736392, force_dump=false) [2025-08-15 10:16:29.766780] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7670-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.768638] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7670-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.768727] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7670-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:29.781018] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.782159] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.783309] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:29.783331] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:29.783343] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.783352] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1003}, replicas:[{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:29.783455] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=80] build ls locality stat map(tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:16:29.783465] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.783476] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.783484] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:29.783493] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:29.783568] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.783580] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.783587] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.783594] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:29.783601] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3720, tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:16:29.783609] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:29.784676] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94106, tenant_id=1004, acc_dr_task=0) [2025-08-15 10:16:29.837772] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC654-0-0] [lt=13] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.851391] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D42-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.881534] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C6-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224189880571000, start_id=1755224189880571000, end_id=1755224189880571001) [2025-08-15 10:16:29.882356] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C6-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224189880571000, v:0}, tenant_id=1004) [2025-08-15 10:16:29.975790] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B39C-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:29.994811] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C590D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:29.995546] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9F0-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.006170] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BF-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1017, thread_idx=0) [2025-08-15 10:16:30.007456] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1017, only_for_display=false) [2025-08-15 10:16:30.008749] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:30.013271] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:30.022651] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.023819] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.024985] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.025008] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1017, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:30.025021] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.025032] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1017, ls_id:{id:1}, replicas:[{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:30.025135] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=80] build ls locality stat map(tenant_id=1017, ls_id={id:1}) [2025-08-15 10:16:30.025143] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.025079] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D3-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.025151] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.025160] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.025172] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:30.025250] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.025264] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.025274] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.025280] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.025288] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3810, tenant_id=1017, ls_id={id:1}) [2025-08-15 10:16:30.025294] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:30.026400] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18944, tenant_id=1017, acc_dr_task=0) [2025-08-15 10:16:30.027617] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D3-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:30.027633] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1018, only_for_display=false) [2025-08-15 10:16:30.027634] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D3-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.028816] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:30.042496] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.043632] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.044803] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.044825] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:30.044842] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.044853] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1}, replicas:[{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:30.044962] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=85] build ls locality stat map(tenant_id=1018, ls_id={id:1}) [2025-08-15 10:16:30.044970] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.044983] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.044995] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.045007] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:30.045089] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.045102] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.045113] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.045123] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.045134] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3784, tenant_id=1018, ls_id={id:1}) [2025-08-15 10:16:30.045144] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:30.046203] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:30.048174] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889F-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.050718] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889F-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:30.050738] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD889F-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.066946] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.068106] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.069254] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.069277] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:30.069293] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.069304] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1001}, replicas:[{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:30.069406] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:16:30.069414] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.069427] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.069437] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.069449] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:30.069528] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.069542] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.069551] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.069556] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.069564] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3779, tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:16:30.069575] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:30.070632] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:30.078493] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB66-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.081042] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB66-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:30.081060] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB66-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.091391] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.092532] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.093671] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.093693] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:30.093709] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.093718] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1002}, replicas:[{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:30.093822] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=81] build ls locality stat map(tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:16:30.093830] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.093843] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.093854] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.093861] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:30.093945] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.093959] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.093968] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.093973] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.093982] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3711, tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:16:30.093994] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:30.095072] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:30.107331] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5907-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.112989] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A1-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224189897338001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224189897338001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224189897338001, v:0}}], last_dump_ts_=1755224185084253, force_dump=false) [2025-08-15 10:16:30.113014] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A1-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.115812] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.115997] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A1-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.116971] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.117901] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A1-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.118056] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A1-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.118141] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:30.118164] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:30.118180] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.118192] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1003}, replicas:[{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:30.118301] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=85] build ls locality stat map(tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:16:30.118309] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.118321] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.118333] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:30.118344] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:30.118426] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.118440] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.118449] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.118454] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:30.118463] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3810, tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:16:30.118470] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:30.119544] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93129, tenant_id=1018, acc_dr_task=0) [2025-08-15 10:16:30.124000] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F1-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:30.126489] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F1-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:30.139494] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0E8-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.151853] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF089-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.188638] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D40-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.203163] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9F5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.211513] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C7-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224190210868000, start_id=1755224190210868000, end_id=1755224190210868001) [2025-08-15 10:16:30.212258] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C7-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224190210868000, v:0}, tenant_id=1007) [2025-08-15 10:16:30.246564] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E913-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:30.249091] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E913-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:30.252510] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5288-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:30.254943] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5288-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:30.288034] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B639A-0-0] [lt=14] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87304688GB}) [2025-08-15 10:16:30.288078] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B639A-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:16:30.318314] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [17739][T1_TimerWK2_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=44] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:30.318347] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [17739][T1_TimerWK2_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=23] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:30.318362] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [17739][T1_TimerWK2_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=12] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:30.318390] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=13] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:30.340637] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5327-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:30.341322] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86BE-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:30.343121] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5327-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:30.343805] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86BE-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:30.408730] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FCD-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:30.411205] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FCD-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:30.444560] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C689F-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.447615] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D1-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:30.450111] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D1-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:30.463093] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:30.463106] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:30.463113] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:30.494227] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227790493014}) [2025-08-15 10:16:30.494245] WDIAG [RS] get_storage_iops_and_bandwidth_limit (ob_shared_storage_net_throt_service.cpp:776) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=-4201] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:0}, max_iops=200, max_bandwidth=25000000, ret=-4201) [2025-08-15 10:16:30.494258] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:30.494271] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:30.500256] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227790499089}) [2025-08-15 10:16:30.500273] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:30.500286] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:30.501470] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227790500299}) [2025-08-15 10:16:30.501486] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:30.501493] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:30.504303] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0E6-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.504363] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E755-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.513704] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF22E-0-0] [lt=10] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.534902] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224188395729001, replayable_scn:1755224188395729001, readable_scn:1755224188395729001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224190531894, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224188395729001, replayable_scn:1755224188395729001, readable_scn:1755224188395729001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224190531894, ora_rowscn_=1755224189765890000, content_changed=true) [2025-08-15 10:16:30.535088] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:30.535098] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:30.535103] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:30.535675] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:30.535686] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:30.535695] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:30.541688] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:30.541702] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:30.541707] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:30.543992] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1391-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83398438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83789062GB}) [2025-08-15 10:16:30.544033] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1391-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:16:30.547236] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:30.547250] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:30.547256] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:30.547266] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1008, cost_time_us=166, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:30.552578] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:30.552589] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:30.552596] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:30.559387] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF98-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.560759] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A40-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.561944] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF98-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:30.561960] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF98-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.563425] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A40-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:30.563442] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A40-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.631520] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:30.632759] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224190631443, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:16:30.632784] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224195632781, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:16:30.693468] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68A2-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.700427] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7713-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.703591] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7713-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.703749] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C48-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.704186] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F2-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.704205] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7881-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.704735] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDB4-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.704789] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07A-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.705599] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7713-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.705685] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7713-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.706821] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C48-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.707245] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7881-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.707303] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F2-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.707681] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07A-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.708951] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C48-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.709080] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C48-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.709241] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7881-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.709248] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F2-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.709327] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7881-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.709344] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F2-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.709392] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:30.709404] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:30.709411] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:30.709663] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.709757] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.738568] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [3414260][T1006_TimerWK4_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=21] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:30.753749] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDC3-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.753869] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF08D-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.754444] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887B-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.756956] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887B-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:30.756971] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887B-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.757298] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5A-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.759843] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5A-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:30.759858] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5A-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:30.769807] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7671-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.772800] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7671-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.774681] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7671-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.774771] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7671-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:30.851459] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D43-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.939924] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=12] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224190549745, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224188395729002, replayable_scn:1755224188395729002, readable_scn:1755224188395729002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:30.956091] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:30.957348] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224190956049, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:16:30.957374] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224195957373, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:16:30.977210] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B39D-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:30.994886] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C590E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:30.995620] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9F1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.030181] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D4-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.032975] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D4-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:31.033001] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D4-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.053512] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A0-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.056974] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A0-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:31.056995] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A0-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.083787] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB67-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.086362] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB67-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:31.086378] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB67-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.087332] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:31.088589] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224191078671, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:16:31.088613] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224196088611, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:16:31.088629] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224196088626, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:31.107409] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5908-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.119041] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A2-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.122081] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A2-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.124175] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A2-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.124300] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A2-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.131900] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F2-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:31.135594] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F2-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:31.139567] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0E9-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.151930] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF08A-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.188674] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D41-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.198969] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=6] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:31.200192] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224191198893, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:16:31.200216] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224196200215, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:16:31.201031] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:31.202200] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224191200963, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:16:31.202220] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224196202219, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:31.203236] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9F6-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.256085] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E914-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:31.259767] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E914-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:31.260253] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5289-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:31.262741] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5289-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:31.348467] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5328-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:31.349122] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86BF-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:31.350953] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5328-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:31.351601] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86BF-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:31.385541] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [727235][T1015_BackupDat][T1015][YB420ACB0122-000633A34EF59998-0-0] [lt=15] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:31.416507] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FCE-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:31.419009] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FCE-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:31.424952] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E54-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224191424904000, start_id=1755224191424904000, end_id=1755224191424904001) [2025-08-15 10:16:31.425815] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E54-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224191424904000, v:0}, tenant_id=1018) [2025-08-15 10:16:31.444630] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68A0-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.455413] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D2-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:31.457897] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D2-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:31.462192] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438128][T1005_DRService][T1005][YB420ACB0122-0006338209C70E47-0-0] [lt=13] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1005, thread_idx=1) [2025-08-15 10:16:31.463307] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:31.463319] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:31.463325] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:31.466006] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [438128][T1005_DRService][T1005][YB420ACB0122-0006338209C70E48-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2485, tenant_id=1005, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:31.468424] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [438128][T1005_DRService][T1005][YB420ACB0122-0006338209C70E48-0-0] [lt=16] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2398, tenant_id=1006, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:31.476898] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E43F-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}) [2025-08-15 10:16:31.476950] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E43F-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=58, ddl_arg=NULL) [2025-08-15 10:16:31.504380] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0E7-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.504435] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E756-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.535167] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:31.535178] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:31.535183] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:31.535781] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:31.535792] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:31.535799] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:31.541598] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224189396868002, replayable_scn:1755224189396868002, readable_scn:1755224189396868002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224191538619, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224189396868002, replayable_scn:1755224189396868002, readable_scn:1755224189396868002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224191538619, ora_rowscn_=1755224190702119000, content_changed=true) [2025-08-15 10:16:31.541775] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:31.541788] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:31.541802] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:31.543093] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:31.547349] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:31.547360] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:31.547368] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:31.553255] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:31.553270] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:31.553275] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:31.564661] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF99-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.566140] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A41-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.567202] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF99-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:31.567221] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF99-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.568698] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A41-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:31.568715] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A41-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.693542] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68A3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.704761] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDB5-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.706725] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7714-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.709858] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7714-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.710010] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C49-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.710210] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7882-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.710323] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F3-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224191399614003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224191399614003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224191399614003, v:0}}], last_dump_ts_=1755224186680256, force_dump=false) [2025-08-15 10:16:31.710347] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F3-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.710846] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.711861] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7714-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.711960] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7714-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.713117] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C49-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.713143] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7882-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.713270] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F3-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224191399614003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224191399614003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224191399614003, v:0}}], last_dump_ts_=1755224186683257, force_dump=false) [2025-08-15 10:16:31.713293] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F3-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.713763] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07B-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224191399614002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224191668990000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224191668990000, v:0}}], last_dump_ts_=1755224186683120, force_dump=false) [2025-08-15 10:16:31.713784] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07B-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.715110] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C49-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.715196] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C49-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.715287] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F3-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.715335] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7882-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.715357] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7882-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.715365] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F3-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.715754] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07B-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.715846] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07B-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.753820] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDC4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.753945] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF08E-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.759645] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887C-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.762214] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887C-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:31.762232] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.762523] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5B-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.765102] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5B-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:31.765118] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5B-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:31.775709] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7672-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.778623] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7672-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.780506] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7672-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.780595] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7672-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:31.790379] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709535][T1014_TimerWK2_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:31.819163] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA4-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224191818297000, start_id=1755224191818297000, end_id=1755224191818297001) [2025-08-15 10:16:31.819981] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA4-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224191818297000, v:0}, tenant_id=1014) [2025-08-15 10:16:31.840299] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:31.841520] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224191840223, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:16:31.841548] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224196841546, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:16:31.842638] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC655-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.850462] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475908][T1008_TimerWK0_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=8] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:31.851532] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D44-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.904697] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [476032][T1007_BackupDat][T1007][YB420ACB0122-0006338212777C1E-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:31.958341] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D6-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1015, thread_idx=0) [2025-08-15 10:16:31.959613] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1015, only_for_display=false) [2025-08-15 10:16:31.960834] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:31.974843] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:31.976006] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:31.977152] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:31.977175] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1015, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:31.977191] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:31.977202] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1015, ls_id:{id:1}, replicas:[{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:31.977311] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=85] build ls locality stat map(tenant_id=1015, ls_id={id:1}) [2025-08-15 10:16:31.977319] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:31.977333] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:31.977344] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:31.977355] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:31.977433] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:31.977447] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:31.977457] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:31.977462] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:31.977471] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3788, tenant_id=1015, ls_id={id:1}) [2025-08-15 10:16:31.977482] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:31.978554] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18942, tenant_id=1015, acc_dr_task=0) [2025-08-15 10:16:31.978585] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B39E-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:31.979781] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1016, only_for_display=false) [2025-08-15 10:16:31.980986] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:31.994756] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:31.994955] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C590F-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.995695] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9F2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:31.995953] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:31.997121] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:31.997144] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:31.997160] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:31.997171] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1}, replicas:[{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:31.997273] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=80] build ls locality stat map(tenant_id=1016, ls_id={id:1}) [2025-08-15 10:16:31.997280] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:31.997294] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:31.997304] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:31.997314] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:31.997392] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:31.997406] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:31.997416] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:31.997422] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:31.997431] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3830, tenant_id=1016, ls_id={id:1}) [2025-08-15 10:16:31.997440] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:31.998501] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:32.019481] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:32.020655] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:32.021837] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:32.021860] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:32.021878] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.021889] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1001}, replicas:[{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:32.022000] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=87] build ls locality stat map(tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:16:32.022008] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:32.022020] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:32.022028] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:32.022038] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:32.022116] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.022130] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.022139] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.022144] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.022153] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3813, tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:16:32.022164] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:32.023252] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:32.035765] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D5-0-0] [lt=41] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.038292] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D5-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:32.038312] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D5-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.044249] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:32.045433] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:32.046588] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:32.046611] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:32.046627] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.046637] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1002}, replicas:[{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:32.046741] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=81] build ls locality stat map(tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:16:32.046749] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:32.046761] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:32.046775] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=11] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:32.046783] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:32.046860] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.046873] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.046882] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.046888] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.046896] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3794, tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:16:32.046903] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:32.048027] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:32.059714] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A1-0-0] [lt=43] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.062259] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A1-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:32.062279] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A1-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.068899] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:32.070050] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:32.071197] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:32.071218] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:32.071229] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=6] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.071237] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=5] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1003}, replicas:[{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:32.071347] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=86] build ls locality stat map(tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:16:32.071357] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:32.071370] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:32.071381] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:32.071392] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:32.071477] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.071492] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.071501] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.071509] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:32.071513] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3765, tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:16:32.071523] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:32.072662] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94092, tenant_id=1016, acc_dr_task=0) [2025-08-15 10:16:32.089098] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB68-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.091599] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB68-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:32.091616] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB68-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.107487] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5909-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.125324] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A3-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.128500] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A3-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.130590] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A3-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.130689] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A3-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.139639] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0EA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.140970] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F3-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:32.143474] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F3-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:32.152013] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF08B-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.184727] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727135][T1016_TimerWK3_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=17] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:32.188710] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D42-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.203308] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9F7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.215998] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727956][T1018_TimerWK0_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=29] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:32.265165] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E915-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:32.267702] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E915-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:32.268171] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD528A-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:32.270316] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:32.270634] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD528A-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:32.288562] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA2F-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87304688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}) [2025-08-15 10:16:32.288606] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA2F-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=50, ddl_arg=NULL) [2025-08-15 10:16:32.330665] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=4] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:32.345241] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:32.346484] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224192345161, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:16:32.346509] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224197346507, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:16:32.356232] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5329-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:32.356796] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C0-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:32.358698] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5329-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:32.359263] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C0-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:32.424326] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FCF-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:32.426839] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FCF-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:32.436826] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:32.438051] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224192436749, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:16:32.438075] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224197438073, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:16:32.439865] INFO [RS] gc_mview (ob_mview_maintenance_task.cpp:267) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=11] mview maintenance task gc mview success(tenant_id_=1008, round_=105551, gc_mview_cost_us_=1220, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:16:32.444707] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68A1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.463027] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224190898703001, replayable_scn:1755224190898703001, readable_scn:1755224190898703001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224192459685, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224190898703001, replayable_scn:1755224190898703001, readable_scn:1755224190898703001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224192459685, ora_rowscn_=1755224192127413000, content_changed=true) [2025-08-15 10:16:32.463213] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:32.463224] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:32.463232] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:32.463290] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D3-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:32.465813] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D3-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:32.504452] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0E8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.504507] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E757-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.513984] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF22F-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.535207] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:32.535221] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:32.535226] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:32.535653] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224190398116002, replayable_scn:1755224190398116002, readable_scn:1755224190398116002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224192532347, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224190398116002, replayable_scn:1755224190398116002, readable_scn:1755224190398116002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224192532347, ora_rowscn_=1755224191712198000, content_changed=true) [2025-08-15 10:16:32.535819] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:32.535831] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:32.535837] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:32.537167] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:32.541247] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:32.541258] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:32.541263] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:32.544461] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6AA9-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83789062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84179688GB}) [2025-08-15 10:16:32.544500] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6AA9-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:16:32.547246] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224190398116002, replayable_scn:1755224190398116002, readable_scn:1755224190398116002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224192543932, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224190398116002, replayable_scn:1755224190398116002, readable_scn:1755224190398116002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224192543932, ora_rowscn_=1755224191712198000, content_changed=true) [2025-08-15 10:16:32.547434] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:32.547445] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:32.547450] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:32.548753] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:32.552473] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224190398116002, replayable_scn:1755224190398116002, readable_scn:1755224190398116002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224192549161, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224190398116002, replayable_scn:1755224190398116002, readable_scn:1755224190398116002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224192549161, ora_rowscn_=1755224191712198000, content_changed=true) [2025-08-15 10:16:32.552701] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:32.552711] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:32.552716] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:32.569928] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9A-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.571410] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A42-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.572443] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9A-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:32.572460] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9A-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.574055] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A42-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:32.574071] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A42-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.693622] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68A4-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.704840] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDB6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.709263] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224190398116002, replayable_scn:1755224190398116002, readable_scn:1755224190398116002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224192705980, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224190398116002, replayable_scn:1755224190398116002, readable_scn:1755224190398116002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224192705980, ora_rowscn_=1755224191712198000, content_changed=true) [2025-08-15 10:16:32.709436] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:32.709450] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:32.709455] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:32.712983] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7715-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.716100] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.716106] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7715-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.716289] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7883-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.716295] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F4-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.716836] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07C-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224192400754001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224192400754002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224192400754002, v:0}}], last_dump_ts_=1755224187686195, force_dump=false) [2025-08-15 10:16:32.716868] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07C-0-0] [lt=25] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.718295] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7715-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.718385] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7715-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.719312] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F4-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.719436] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7883-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224192400754002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224192400754003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224192400754003, v:0}}], last_dump_ts_=1755224187688775, force_dump=false) [2025-08-15 10:16:32.719460] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7883-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.719673] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4A-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.720313] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.721310] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F4-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.721356] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7883-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.721414] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F4-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.721432] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7883-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.721653] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.721732] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.722256] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.722339] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.753892] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDC5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.754016] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF08F-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.764921] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887D-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.767488] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887D-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:32.767507] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887D-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.767768] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5C-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.770284] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5C-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:32.770302] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:32.777353] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=13] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:32.777388] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=23] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:32.777403] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=10] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:32.778572] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [727135][T1016_TimerWK3_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=17] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:32.781582] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7673-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.784561] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7673-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.786439] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7673-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.786524] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7673-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:32.810179] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:32.811412] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224192810101, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:16:32.811437] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224197811435, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:16:32.812605] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224192811281, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:16:32.812624] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224197812622, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:16:32.851607] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D45-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.973495] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:32.974770] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224192973414, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:16:32.974791] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224197974789, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:16:32.979998] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B39F-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:32.995030] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5910-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:32.995770] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9F3-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.040534] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D6-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.043120] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D6-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:33.043138] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D6-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.048916] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69ED-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1013, thread_idx=0) [2025-08-15 10:16:33.050176] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1013, only_for_display=false) [2025-08-15 10:16:33.051433] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:33.059095] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:33.060356] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224193059016, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:16:33.060381] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224198060379, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:16:33.064987] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A2-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.065408] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.066578] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.067525] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A2-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:33.067543] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A2-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.067812] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.067836] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1013, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:33.067853] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.067865] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1013, ls_id:{id:1}, replicas:[{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:33.067982] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=92] build ls locality stat map(tenant_id=1013, ls_id={id:1}) [2025-08-15 10:16:33.067991] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.068004] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.068014] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.068026] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:33.068103] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.068118] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.068127] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.068133] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.068141] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3895, tenant_id=1013, ls_id={id:1}) [2025-08-15 10:16:33.068148] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:33.069284] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19107, tenant_id=1013, acc_dr_task=0) [2025-08-15 10:16:33.070506] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1014, only_for_display=false) [2025-08-15 10:16:33.071728] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:33.085697] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.086882] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.088085] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.088107] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:33.088124] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.088135] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1}, replicas:[{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:33.088239] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=81] build ls locality stat map(tenant_id=1014, ls_id={id:1}) [2025-08-15 10:16:33.088247] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.088260] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.088271] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.088282] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:33.088360] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.088373] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.088383] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.088388] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.088397] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3899, tenant_id=1014, ls_id={id:1}) [2025-08-15 10:16:33.088407] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:33.089517] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:33.094389] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB69-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.096929] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB69-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:33.096953] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB69-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.107526] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C590A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.110587] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.111783] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.112948] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.112972] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:33.112990] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.113001] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1001}, replicas:[{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:33.113123] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=96] build ls locality stat map(tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:16:33.113132] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.113145] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.113151] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.113162] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:33.113240] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.113254] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.113263] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.113269] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.113277] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3834, tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:16:33.113288] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:33.114376] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:33.131711] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A4-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.134393] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:340) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A4-0-0] [lt=14] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1014, ls_id:{id:1}, sync_scn:{val:1755224192901554002, v:0}, readable_scn:{val:1755224192901554002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-08-15 10:16:33.134780] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A4-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.135454] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.136626] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.136878] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A4-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.137078] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A4-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.137117] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:459) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A4-0-0] [lt=6] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1755224190898703001, v:0}) [2025-08-15 10:16:33.137816] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.137840] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:33.137858] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.137870] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1002}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:33.137992] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=96] build ls locality stat map(tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:16:33.138000] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.138013] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.138024] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.138035] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:33.138112] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.138126] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.138136] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.138142] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.138151] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3863, tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:16:33.138161] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:33.139238] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:33.139713] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0EB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.148776] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F4-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:33.151269] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F4-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:33.152084] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF08C-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.160183] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.161345] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.162502] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:33.162527] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:33.162543] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.162553] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1003}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:33.162670] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=91] build ls locality stat map(tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:16:33.162679] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.162692] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.162702] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:33.162713] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:33.162789] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.162803] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.162812] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.162819] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:33.162827] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3795, tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:16:33.162838] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:33.163936] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94636, tenant_id=1014, acc_dr_task=0) [2025-08-15 10:16:33.188786] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D43-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.203377] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9F8-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.273146] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E916-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:33.275671] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E916-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:33.275948] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD528B-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:33.279522] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD528B-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:33.364009] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD532A-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:33.364578] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C1-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:33.366501] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD532A-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:33.367082] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C1-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:33.432735] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD0-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:33.436399] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD0-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:33.444784] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68A2-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.463290] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:33.463302] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:33.463307] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:33.471147] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D4-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:33.473666] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D4-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:33.477398] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D52-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}) [2025-08-15 10:16:33.477442] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D52-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:16:33.504527] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0E9-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.504581] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E758-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.505773] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227793504591}) [2025-08-15 10:16:33.505788] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:33.505799] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:33.511670] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227793510503}) [2025-08-15 10:16:33.511684] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:33.511695] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:33.512866] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227793511706}) [2025-08-15 10:16:33.512878] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:33.512888] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:33.535119] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224191399614004, replayable_scn:1755224191399614004, readable_scn:1755224191399614004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224193532457, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224191399614004, replayable_scn:1755224191399614004, readable_scn:1755224191399614004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224193532457, ora_rowscn_=1755224192783828000, content_changed=true) [2025-08-15 10:16:33.535308] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:33.535327] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:33.535338] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:33.535904] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:33.535915] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:33.535923] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:33.541272] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:33.541287] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:33.541293] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:33.547484] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:33.547495] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:33.547503] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:33.547513] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1008, cost_time_us=179, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:33.552679] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:33.552698] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:33.552704] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:33.575235] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9B-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.576757] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A43-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.577740] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9B-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:33.577758] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.579326] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A43-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:33.579341] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A43-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.617230] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:33.618481] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224193617161, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:16:33.618503] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224198618502, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:16:33.693707] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68A5-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.704914] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDB7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.709556] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:33.709568] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:33.709573] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:33.719322] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7716-0-0] [lt=5] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224193402020001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224193402020002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224193402020002, v:0}}], last_dump_ts_=1755224188687310, force_dump=false) [2025-08-15 10:16:33.719349] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7716-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.722409] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F5-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.722448] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7884-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.722533] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7716-0-0] [lt=16] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224193402020001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224193402020002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224193402020002, v:0}}], last_dump_ts_=1755224188690379, force_dump=false) [2025-08-15 10:16:33.722561] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7716-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.722576] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4B-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.723249] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07D-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.724487] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7716-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.724577] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7716-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.725226] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F5-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.725446] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7884-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.726038] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4B-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.726712] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.727227] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F5-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.727320] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F5-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.727359] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7884-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.727519] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7884-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.727979] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4B-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.728091] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4B-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.728612] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.728800] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.753950] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDC6-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.754088] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF090-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.770226] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.772794] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887E-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:33.772810] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.772988] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5D-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.775512] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5D-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:33.775531] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5D-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:33.787513] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7674-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.790484] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7674-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.792554] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7674-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.792653] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7674-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:33.847532] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC656-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.851679] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D46-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.941598] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224193549318, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224191668990000, replayable_scn:1755224191668990000, readable_scn:1755224191668990000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:33.981413] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3A0-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:33.995104] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5911-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:33.995844] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9F4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.045278] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D7-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.047831] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D7-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:34.047851] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D7-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.070208] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A3-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.072708] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A3-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:34.072725] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A3-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.099688] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6A-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.102201] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6A-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:34.102219] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.107601] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C590B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.138083] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A5-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.139785] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0EC-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.141210] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A5-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224193902765001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224193902765001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224193902765001, v:0}}], last_dump_ts_=1755224189109985, force_dump=false) [2025-08-15 10:16:34.141232] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A5-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.143184] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A5-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.143293] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A5-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.152161] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF08D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.156681] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F5-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:34.159202] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F5-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:34.188862] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D44-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.203447] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9F9-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.280981] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E917-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:34.283490] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E917-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:34.284753] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD528C-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:34.287297] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD528C-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:34.289016] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA33-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}) [2025-08-15 10:16:34.289060] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA33-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:16:34.372255] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD532B-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:34.372366] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C2-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:34.374947] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C2-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:34.376645] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD532B-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:34.441717] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD1-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:34.444236] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD1-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:34.444857] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68A3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.463359] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:34.463371] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:34.463376] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:34.464703] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:34.478989] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D5-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:34.480167] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D5-0-0] [lt=13] log restore source is empty, just skip(ret=-4018, user_tenant_id=1008) [2025-08-15 10:16:34.482602] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D5-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:34.504600] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0EA-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.504653] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E759-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.514273] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF230-0-0] [lt=10] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.535277] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:34.535294] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:34.535299] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:34.535927] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:34.535943] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:34.535952] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:34.536593] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:34.541309] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:34.541330] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:34.541336] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:34.544931] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAC5-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84179688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84570312GB}) [2025-08-15 10:16:34.544974] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAC5-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:16:34.547421] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:34.547432] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:34.547439] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:34.552825] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:34.552837] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:34.552843] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:34.554207] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:34.580500] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9C-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.582018] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A44-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.583050] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9C-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:34.583067] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.584550] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A44-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:34.584567] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A44-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.623516] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:104) [17458][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=14] start checking lost replicas [2025-08-15 10:16:34.637758] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:34.638975] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224194637679, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:16:34.638999] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224199638997, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:16:34.693784] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68A6-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.704992] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDB8-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.709541] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:34.709555] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:34.709561] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:34.710883] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:34.725576] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7717-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.728219] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F6-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.728476] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7885-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224194403500003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224194403500003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224194403500003, v:0}}], last_dump_ts_=1755224189698019, force_dump=false) [2025-08-15 10:16:34.728507] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7885-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.728708] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7717-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.729018] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4C-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224194403500001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224194403500002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224194403500002, v:0}}], last_dump_ts_=1755224189697608, force_dump=false) [2025-08-15 10:16:34.729043] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.729758] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.730677] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7717-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.730781] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7717-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.731274] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F6-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.731460] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7885-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.732711] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4C-0-0] [lt=18] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224194403500001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224194403500002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224194403500002, v:0}}], last_dump_ts_=1755224189700598, force_dump=false) [2025-08-15 10:16:34.732742] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4C-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.733218] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F6-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.733307] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F6-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.733331] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07E-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.733346] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7885-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.733429] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7885-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.735176] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.735293] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4C-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.735732] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.735822] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07E-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.754021] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDC7-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.754158] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF091-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.775519] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887F-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.778085] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887F-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:34.778105] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D887F-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.778170] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.780667] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5E-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:34.780684] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5E-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:34.793627] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7675-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224194518178000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224194518178001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224194518178001, v:0}}], last_dump_ts_=1755224189763688, force_dump=false) [2025-08-15 10:16:34.793650] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7675-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.796563] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7675-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224194518178001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224194768501000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224194768501000, v:0}}], last_dump_ts_=1755224189766754, force_dump=false) [2025-08-15 10:16:34.796585] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7675-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.798495] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7675-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.798588] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7675-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:34.851753] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D47-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.982823] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3A1-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:34.995175] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5912-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:34.995918] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9F5-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.014566] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=5] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:35.029869] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [2619488][T1012_TimerWK3_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=8] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:35.029902] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [2619488][T1012_TimerWK3_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=22] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:35.029917] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [2619488][T1012_TimerWK3_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:35.031174] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [2619486][T1012_TimerWK1_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=14] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:35.050004] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D8-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.052515] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D8-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:35.052531] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D8-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.075426] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A4-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.077968] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A4-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:35.077986] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A4-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.105007] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6B-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.107543] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6B-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:35.107559] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6B-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.107678] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C590C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.127795] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [1487708][T1003_ArchiveSv][T1003][YB420ACB0122-0006337FE36598F2-0-0] [lt=8] start do ObArchiveSchedulerService round(round=186151) [2025-08-15 10:16:35.131435] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [1487708][T1003_ArchiveSv][T1003][YB420ACB0122-0006337FE36598F2-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:16:35.131454] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [1487708][T1003_ArchiveSv][T1003][YB420ACB0122-0006337FE36598F2-0-0] [lt=16][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1004 AND dest_no = 0 AND name = 'state', exec_tenant_id=1003) [2025-08-15 10:16:35.131501] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [1487708][T1003_ArchiveSv][T1003][YB420ACB0122-0006337FE36598F2-0-0] [lt=11][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1004, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:16:35.131516] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [1487708][T1003_ArchiveSv][T1003][YB420ACB0122-0006337FE36598F2-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1004, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:16:35.131529] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [1487708][T1003_ArchiveSv][T1003][YB420ACB0122-0006337FE36598F2-0-0] [lt=9][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1004, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:16:35.139860] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0ED-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.144341] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A6-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224194904248001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224194904248001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224194904248001, v:0}}], last_dump_ts_=1755224190112988, force_dump=false) [2025-08-15 10:16:35.144365] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A6-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.147490] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A6-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.149443] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A6-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.149552] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A6-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.152236] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF08E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.164634] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F6-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:35.167148] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:35.188935] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D45-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.203518] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9FA-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.288922] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E918-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:35.291446] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E918-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:35.292608] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD528D-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:35.295064] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD528D-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:35.380294] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C3-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:35.381851] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD532C-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:35.382797] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C3-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:35.384286] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD532C-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:35.444931] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68A4-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.449541] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD2-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:35.452031] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD2-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:35.463261] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224193902765001, replayable_scn:1755224193902765001, readable_scn:1755224193902765001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224195460395, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224193902765001, replayable_scn:1755224193902765001, readable_scn:1755224193902765001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224195460395, ora_rowscn_=1755224195146153000, content_changed=true) [2025-08-15 10:16:35.463471] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:35.463485] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:35.463490] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:35.477823] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E451-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}) [2025-08-15 10:16:35.477869] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E451-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=52, ddl_arg=NULL) [2025-08-15 10:16:35.488558] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D6-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:35.491038] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:35.495313] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5B-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1011, thread_idx=0) [2025-08-15 10:16:35.496616] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1011, only_for_display=false) [2025-08-15 10:16:35.497891] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:35.504673] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0EB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.504727] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E75A-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.513293] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.514447] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.515609] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.515633] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1011, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:35.515650] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.515662] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1011, ls_id:{id:1}, replicas:[{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:35.515779] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=91] build ls locality stat map(tenant_id=1011, ls_id={id:1}) [2025-08-15 10:16:35.515788] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.515801] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.515808] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.515820] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:35.515899] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.515915] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.515925] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.515933] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.515946] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=10] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3855, tenant_id=1011, ls_id={id:1}) [2025-08-15 10:16:35.515952] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:35.517489] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=20873, tenant_id=1011, acc_dr_task=0) [2025-08-15 10:16:35.518718] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1012, only_for_display=false) [2025-08-15 10:16:35.519910] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:35.535046] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.535304] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:35.535316] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:35.535326] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:35.535804] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224193402020003, replayable_scn:1755224193402020003, readable_scn:1755224193402020003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224195532995, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224193402020003, replayable_scn:1755224193402020003, readable_scn:1755224193402020003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224195532995, ora_rowscn_=1755224194729549000, content_changed=true) [2025-08-15 10:16:35.535990] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:35.536001] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:35.536006] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:35.536200] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.537372] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.537394] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:35.537407] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.537417] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1}, replicas:[{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:35.537526] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=84] build ls locality stat map(tenant_id=1012, ls_id={id:1}) [2025-08-15 10:16:35.537534] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.537547] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.537556] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.537566] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:35.537644] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.537656] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.537665] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.537673] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.537681] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3794, tenant_id=1012, ls_id={id:1}) [2025-08-15 10:16:35.537689] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:35.539227] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:35.541186] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224193402020002, replayable_scn:1755224193402020002, readable_scn:1755224193402020002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224195538263, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224193402020002, replayable_scn:1755224193402020002, readable_scn:1755224193402020002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224195538263, ora_rowscn_=1755224194727446000, content_changed=true) [2025-08-15 10:16:35.541341] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:35.541352] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:35.541356] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:35.547318] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224193402020002, replayable_scn:1755224193402020002, readable_scn:1755224193402020002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224195544565, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224193402020002, replayable_scn:1755224193402020002, readable_scn:1755224193402020002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224195544565, ora_rowscn_=1755224194731655000, content_changed=true) [2025-08-15 10:16:35.547511] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:35.547528] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:35.547537] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:35.552773] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224193402020003, replayable_scn:1755224193402020003, readable_scn:1755224193402020003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224195549832, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224193402020003, replayable_scn:1755224193402020003, readable_scn:1755224193402020003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224195549832, ora_rowscn_=1755224194731655001, content_changed=true) [2025-08-15 10:16:35.552956] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:35.552967] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:35.552974] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:35.561355] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.562495] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.563640] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.563667] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:35.563684] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.563695] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1001}, replicas:[{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:35.563805] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=85] build ls locality stat map(tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:16:35.563814] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.563829] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=12] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.563839] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.563847] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:35.563927] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.563947] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.563957] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.563962] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.563971] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3755, tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:16:35.563980] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:35.565489] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:35.585782] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9D-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.587301] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A45-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.587912] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.588411] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9D-0-0] [lt=70] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:35.588428] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9D-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.589066] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.589833] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A45-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:35.589850] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A45-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.590232] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.590254] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:35.590272] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.590282] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1002}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:35.590389] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=83] build ls locality stat map(tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:16:35.590400] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.590412] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.590421] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.590431] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:35.590509] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.590522] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.590531] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.590539] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.590546] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3802, tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:16:35.590555] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:35.592069] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:35.614397] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.615545] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.616695] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.616722] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:35.616738] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.616747] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1003}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:35.616856] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=84] build ls locality stat map(tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:16:35.616866] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.616878] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.616886] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.616896] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:35.616980] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.616991] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.616999] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.617008] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.617013] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3767, tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:16:35.617023] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:35.618512] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=101006, tenant_id=1012, acc_dr_task=0) [2025-08-15 10:16:35.618876] INFO [RS.LB] idle (ob_root_balancer.cpp:87) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=29] root balance idle(idle_time=10000000) [2025-08-15 10:16:35.618896] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:70) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=15] start check unit migrate finish [2025-08-15 10:16:35.618910] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1) [2025-08-15 10:16:35.628161] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=7] try check migrate unit finish by tenant(tenant_id=1004) [2025-08-15 10:16:35.632855] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:35.634095] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224195632781, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:16:35.634120] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224200634118, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:16:35.653784] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1006) [2025-08-15 10:16:35.679404] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1008) [2025-08-15 10:16:35.693859] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68A7-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.705070] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDB9-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.705821] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1012) [2025-08-15 10:16:35.709458] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224193402020004, replayable_scn:1755224193402020004, readable_scn:1755224193402020004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224195706614, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224193402020004, replayable_scn:1755224193402020004, readable_scn:1755224193402020004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224195706614, ora_rowscn_=1755224194730601000, content_changed=true) [2025-08-15 10:16:35.709644] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:35.709658] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:35.709663] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:35.731352] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=17] try check migrate unit finish by tenant(tenant_id=1014) [2025-08-15 10:16:35.731899] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7718-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.734245] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F7-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.734358] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7886-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.734960] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7718-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.736229] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.736774] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07F-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.736885] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7718-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.737013] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7718-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.737336] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7886-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.737417] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F7-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.739276] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4D-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.739369] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F7-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.739426] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7886-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.739467] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F7-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.739523] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7886-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.739754] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07F-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.741226] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.741433] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.741779] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.741862] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC07F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.743645] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438023][T1006_TimerWK2_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=8] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:35.754095] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDC8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.754231] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF092-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.757841] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=16] try check migrate unit finish by tenant(tenant_id=1016) [2025-08-15 10:16:35.780814] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8880-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.783378] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8880-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:35.783408] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=12] try check migrate unit finish by tenant(tenant_id=1018) [2025-08-15 10:16:35.783393] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8880-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.783380] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5F-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.785911] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5F-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:35.785927] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A5F-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:35.787867] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED9-0-0] [lt=12] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1007, thread_idx=0) [2025-08-15 10:16:35.789138] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1007, only_for_display=false) [2025-08-15 10:16:35.790343] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:35.799585] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7676-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.802600] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7676-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.804236] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.804502] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7676-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.804585] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7676-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:35.805388] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.806566] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.806593] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1007, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:35.806605] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.806612] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=5] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1007, ls_id:{id:1}, replicas:[{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:35.806713] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=79] build ls locality stat map(tenant_id=1007, ls_id={id:1}) [2025-08-15 10:16:35.806720] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.806731] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.806739] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.806748] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:35.806826] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.806838] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.806846] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.806853] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.806859] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3802, tenant_id=1007, ls_id={id:1}) [2025-08-15 10:16:35.806867] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:35.807947] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18809, tenant_id=1007, acc_dr_task=0) [2025-08-15 10:16:35.808987] INFO [RS] check (ob_alter_locality_finish_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81594-0-0] [lt=13] start to check alter locality finish [2025-08-15 10:16:35.809169] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1008, only_for_display=false) [2025-08-15 10:16:35.810129] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81595-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.810358] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:35.811318] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81595-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.813608] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81597-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.814750] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81597-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.817055] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81599-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.818212] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81599-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.820482] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8159B-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.821608] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8159B-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.823906] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8159D-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.824152] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.825054] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8159D-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.825317] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.826460] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.826489] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:35.826506] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.826517] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1}, replicas:[{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:35.826619] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=79] build ls locality stat map(tenant_id=1008, ls_id={id:1}) [2025-08-15 10:16:35.826626] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.826639] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.826649] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.826661] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:35.826745] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=63] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.826759] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.826769] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.826776] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.826784] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3805, tenant_id=1008, ls_id={id:1}) [2025-08-15 10:16:35.826795] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:35.827337] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8159F-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.827863] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:35.828468] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8159F-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.830779] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A1-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.831922] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A1-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.834180] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A3-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.835342] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A3-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.836520] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A3-0-0] [lt=12] start check shrink resource pool [2025-08-15 10:16:35.836537] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A4-0-0] [lt=11] start check shrink resource pool(tenant_id=1) [2025-08-15 10:16:35.837643] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A4-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.838771] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A4-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.838823] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A5-0-0] [lt=13] start check shrink resource pool(tenant_id=1004) [2025-08-15 10:16:35.839931] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A5-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.841071] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A5-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.841121] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A6-0-0] [lt=12] start check shrink resource pool(tenant_id=1006) [2025-08-15 10:16:35.842262] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A6-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.843399] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A6-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.843447] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A7-0-0] [lt=12] start check shrink resource pool(tenant_id=1008) [2025-08-15 10:16:35.844550] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A7-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.845702] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A7-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.845751] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A8-0-0] [lt=11] start check shrink resource pool(tenant_id=1012) [2025-08-15 10:16:35.846857] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A8-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.848013] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A8-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.848062] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A9-0-0] [lt=12] start check shrink resource pool(tenant_id=1014) [2025-08-15 10:16:35.848725] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.849164] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A9-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.849891] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.850305] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815A9-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.850354] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AA-0-0] [lt=12] start check shrink resource pool(tenant_id=1016) [2025-08-15 10:16:35.851046] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.851069] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:35.851087] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.851098] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1001}, replicas:[{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:35.851201] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=80] build ls locality stat map(tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:16:35.851209] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.851222] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.851232] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.851240] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:35.851321] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=59] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.851335] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.851344] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.851349] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.851357] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3788, tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:16:35.851363] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:35.851456] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AA-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.851825] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D48-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.852360] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC657-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.852424] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:35.852588] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AA-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.852637] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AB-0-0] [lt=12] start check shrink resource pool(tenant_id=1018) [2025-08-15 10:16:35.853737] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AB-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.854867] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AB-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.854917] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:99) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AB-0-0] [lt=12] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.854930] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AB-0-0] [lt=9] start check primary zone [2025-08-15 10:16:35.854950] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AC-0-0] [lt=13] start check primary zone(tenant_id=1) [2025-08-15 10:16:35.856064] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AC-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.856114] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AD-0-0] [lt=12] start check primary zone(tenant_id=1004) [2025-08-15 10:16:35.857222] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AD-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.857270] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AE-0-0] [lt=11] start check primary zone(tenant_id=1006) [2025-08-15 10:16:35.858365] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AE-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.858412] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AF-0-0] [lt=11] start check primary zone(tenant_id=1008) [2025-08-15 10:16:35.859515] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815AF-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.859577] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B0-0-0] [lt=12] start check primary zone(tenant_id=1012) [2025-08-15 10:16:35.860683] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B0-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.860731] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B1-0-0] [lt=11] start check primary zone(tenant_id=1014) [2025-08-15 10:16:35.861820] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B1-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.861867] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B2-0-0] [lt=12] start check primary zone(tenant_id=1016) [2025-08-15 10:16:35.862982] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B2-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.863031] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=12] start check primary zone(tenant_id=1018) [2025-08-15 10:16:35.864136] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:35.864186] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:166) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=13] start do unit balance [2025-08-15 10:16:35.871050] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:350) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=10] start distribute for server status change [2025-08-15 10:16:35.871150] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:401) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=11] finish distribute for server status change(ret=0) [2025-08-15 10:16:35.871160] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:200) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=8] finish do unit balance(ret=0) [2025-08-15 10:16:35.873230] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.874423] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=25] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.875605] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.875630] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:35.875642] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.875654] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1002}, replicas:[{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:35.875768] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=88] build ls locality stat map(tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:16:35.875778] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.875791] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.875801] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.875814] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:35.875892] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.875907] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.875918] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.875924] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.875934] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3871, tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:16:35.875945] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:35.877011] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:35.877138] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:115) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=5] start to do tenant group unit balance [2025-08-15 10:16:35.877182] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=13] start to do tenant group unit balance(zone=zone1) [2025-08-15 10:16:35.877196] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=8] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.203.1.34:2882", disk_in_use_:3110076416, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:35.877214] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=12] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:35.877243] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=10] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:35.877261] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=8] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:35.877271] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone1) [2025-08-15 10:16:35.877282] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=7] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:35.877303] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=14] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.1.34:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:35.877346] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=34] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.203.1.34:2882", disk_in_use_:3110076416, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:35.877357] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=9] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.1.34:2882", disk_in_use_:3110076416, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:35.877375] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=12] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:35.877388] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=10] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1006, unit_config:{unit_config_id:1005, name:"config_opsone_zone1_s_2_4_ywx", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7988, unit_config:0x7f1e9aa71c38, pool:0x7f1c3beb2f18}, {unit_id:1008, tenant_id:1006, resource_pool_id:1008, unit_config:{unit_config_id:1008, name:"config_test_zone1_s_2_4_emp", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f77d8, unit_config:0x7f1e9aa71998, pool:0x7f1c3beb27a8}, {unit_id:1011, tenant_id:1008, resource_pool_id:1011, unit_config:{unit_config_id:1011, name:"config_mcp_zone1_s_4_8_tmc", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7550, unit_config:0x7f1e9aa716f8, pool:0x7f1e443fda40}, {unit_id:1016, tenant_id:1012, resource_pool_id:1016, unit_config:{unit_config_id:1017, name:"config_oms_zone1_s_4_8_gzp", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f73a0, unit_config:0x7f1e9aa71458, pool:0x7f1e443fd2d0}, {unit_id:1019, tenant_id:1014, resource_pool_id:1019, unit_config:{unit_config_id:1019, name:"config_grafana_test_zone1_s_1_2_pri", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7118, unit_config:0x7f1e9aa71298, pool:0x7f1e443fc7a8}, {unit_id:1024, tenant_id:1016, resource_pool_id:1024, unit_config:{unit_config_id:1023, name:"config_temporalio_zone1_s_2_4_ykw", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6ce0, unit_config:0x7f1e9aa70f18, pool:0x7f1e74dff2d0}, {unit_id:1026, tenant_id:1018, resource_pool_id:1026, unit_config:{unit_config_id:1027, name:"config_media_zone1_s_2_4_kmn", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6b30, unit_config:0x7f1e9aa70b98, pool:0x7f1e74dfeb60}], not_grant_units=[], available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:35.877446] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=35] start to do tenant group unit balance(zone=zone2) [2025-08-15 10:16:35.877455] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=7] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.203.4.38:2882", disk_in_use_:3093299200, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:35.877467] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:35.877483] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=7] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:35.877495] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=6] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:35.877503] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone2) [2025-08-15 10:16:35.877507] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:35.877521] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=10] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.38:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:35.877554] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=25] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.203.4.38:2882", disk_in_use_:3093299200, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:35.877563] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=7] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.38:2882", disk_in_use_:3093299200, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:35.877572] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=6] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:35.877583] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=8] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1004, unit_config:{unit_config_id:1004, name:"config_opsone_zone2_s_2_4_wag", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7b38, unit_config:0x7f1e9aa71d18, pool:0x7f1c3beb3688}, {unit_id:1009, tenant_id:1006, resource_pool_id:1009, unit_config:{unit_config_id:1009, name:"config_test_zone2_s_2_4_yey", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7700, unit_config:0x7f1e9aa718b8, pool:0x7f1c3beb23f0}, {unit_id:1010, tenant_id:1008, resource_pool_id:1010, unit_config:{unit_config_id:1010, name:"config_mcp_zone2_s_4_8_bjb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7628, unit_config:0x7f1e9aa717d8, pool:0x7f1c3beb2038}, {unit_id:1018, tenant_id:1012, resource_pool_id:1018, unit_config:{unit_config_id:1016, name:"config_oms_zone2_s_4_8_kbx", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f71f0, unit_config:0x7f1e9aa71538, pool:0x7f1e443fcb60}, {unit_id:1021, tenant_id:1014, resource_pool_id:1021, unit_config:{unit_config_id:1021, name:"config_grafana_test_zone2_s_1_2_yyc", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f6f68, unit_config:0x7f1e9aa710d8, pool:0x7f1e443fc038}, {unit_id:1023, tenant_id:1016, resource_pool_id:1023, unit_config:{unit_config_id:1022, name:"config_temporalio_zone2_s_2_4_lsv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6db8, unit_config:0x7f1e9aa70ff8, pool:0x7f1e74dff688}, {unit_id:1027, tenant_id:1018, resource_pool_id:1027, unit_config:{unit_config_id:1025, name:"config_media_zone2_s_2_4_yzd", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6a58, unit_config:0x7f1e9aa70d58, pool:0x7f1e74dfe7a8}], not_grant_units=[], available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:35.877634] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=30] start to do tenant group unit balance(zone=zone3) [2025-08-15 10:16:35.877642] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.203.4.39:2882", disk_in_use_:3055550464, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:35.877654] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:35.877669] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=8] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:35.877681] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=6] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:35.877690] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone3) [2025-08-15 10:16:35.877694] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:35.877706] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84570312GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.39:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84570312GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:35.877737] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=25] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.203.4.39:2882", disk_in_use_:3055550464, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:35.877745] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=6] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.39:2882", disk_in_use_:3055550464, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:35.877756] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:35.877766] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=7] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1005, unit_config:{unit_config_id:1006, name:"config_opsone_zone3_s_2_4_vxv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7a60, unit_config:0x7f1e9aa71b58, pool:0x7f1c3beb32d0}, {unit_id:1007, tenant_id:1006, resource_pool_id:1007, unit_config:{unit_config_id:1007, name:"config_test_zone3_s_2_4_mui", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f78b0, unit_config:0x7f1e9aa71a78, pool:0x7f1c3beb2b60}, {unit_id:1012, tenant_id:1008, resource_pool_id:1012, unit_config:{unit_config_id:1012, name:"config_mcp_zone3_s_4_8_seu", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7478, unit_config:0x7f1e9aa71618, pool:0x7f1e443fd688}, {unit_id:1017, tenant_id:1012, resource_pool_id:1017, unit_config:{unit_config_id:1018, name:"config_oms_zone3_s_4_8_nvb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f72c8, unit_config:0x7f1e9aa71378, pool:0x7f1e443fcf18}, {unit_id:1020, tenant_id:1014, resource_pool_id:1020, unit_config:{unit_config_id:1020, name:"config_grafana_test_zone3_s_1_2_xbz", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7040, unit_config:0x7f1e9aa711b8, pool:0x7f1e443fc3f0}, {unit_id:1022, tenant_id:1016, resource_pool_id:1022, unit_config:{unit_config_id:1024, name:"config_temporalio_zone3_s_2_4_rxz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6e90, unit_config:0x7f1e9aa70e38, pool:0x7f1e74dffa40}, {unit_id:1025, tenant_id:1018, resource_pool_id:1025, unit_config:{unit_config_id:1026, name:"config_media_zone3_s_2_4_ukz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6c08, unit_config:0x7f1e9aa70c78, pool:0x7f1e74dfef18}], not_grant_units=[], available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:35.898276] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.899448] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.900595] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:35.900619] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:35.900636] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.900648] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1003}, replicas:[{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:35.900753] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=81] build ls locality stat map(tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:16:35.900767] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.900781] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.900790] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:35.900803] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:35.900880] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.900894] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.900904] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.900909] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:35.900918] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3796, tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:16:35.900928] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:35.902017] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94057, tenant_id=1008, acc_dr_task=0) [2025-08-15 10:16:35.957447] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:35.958646] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224195957373, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:16:35.958670] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224200958669, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:16:35.984270] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3A2-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:35.995251] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5913-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:35.995992] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9F6-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.054710] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D9-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.057254] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D9-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:36.057273] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95D9-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.080684] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A5-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.083253] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A5-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:36.083271] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A5-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.097244] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:36.098504] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224196088611, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:16:36.098529] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224201098527, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:16:36.098541] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224201098539, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:36.107752] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C590D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.110311] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6C-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.112899] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6C-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:36.112915] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6C-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.139937] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0EE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.150597] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A7-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.152311] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF08F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.153577] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A7-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.155512] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A7-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.155600] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A7-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.172621] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F7-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:36.175150] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F7-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:36.189013] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D46-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.200289] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:36.201528] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224196200215, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:16:36.201551] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224201201550, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:16:36.202274] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:36.203431] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224196202219, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:16:36.203453] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224201203453, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:36.203592] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9FB-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.287383] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [709536][T1014_TimerWK3_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=14] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:36.287420] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [709536][T1014_TimerWK3_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=24] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:36.287436] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [709536][T1014_TimerWK3_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=12] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:36.289521] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B63A5-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}) [2025-08-15 10:16:36.289564] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B63A5-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:16:36.289885] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [709535][T1014_TimerWK2_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=29] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:36.296786] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E919-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:36.299304] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E919-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:36.300355] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD528E-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:36.302820] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD528E-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:36.388186] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C4-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:36.389520] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD532D-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:36.390704] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C4-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:36.391992] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD532D-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:36.445017] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68A5-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.457407] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD3-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:36.459904] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD3-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:36.463429] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:36.463449] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:36.463456] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:36.496375] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D7-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:36.498928] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D7-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:36.504751] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0EC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.504798] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E75B-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.514539] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF231-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.517242] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227796516043}) [2025-08-15 10:16:36.517259] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:36.517272] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:36.523217] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227796522036}) [2025-08-15 10:16:36.523234] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:36.523242] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:36.524438] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227796523252}) [2025-08-15 10:16:36.524454] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:36.524461] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:36.535264] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224194584381003, replayable_scn:1755224194584381003, readable_scn:1755224194584381003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224196532396, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224194584381003, replayable_scn:1755224194584381003, readable_scn:1755224194584381003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224196532396, ora_rowscn_=1755224195801242000, content_changed=true) [2025-08-15 10:16:36.535468] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:36.535482] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:36.535494] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:36.536195] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:36.536216] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:36.536228] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:36.541442] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:36.541454] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:36.541459] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:36.545423] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBACB-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84570312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84960938GB}) [2025-08-15 10:16:36.545462] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBACB-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:16:36.547447] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:36.547459] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:36.547466] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:36.552922] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:36.552934] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:36.552946] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:36.552955] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1006, cost_time_us=181, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:36.591139] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9E-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.592521] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A46-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.593686] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9E-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:36.593703] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9E-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.595058] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A46-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:36.595075] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A46-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.693933] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68A8-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.705147] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDBA-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.709739] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:36.709753] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:36.709758] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:36.738074] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7719-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.740443] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F8-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224196406172002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224196406172002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224196406172002, v:0}}], last_dump_ts_=1755224191710322, force_dump=false) [2025-08-15 10:16:36.740467] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F8-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.740527] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7887-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.741640] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7719-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.742345] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4E-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.742734] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC080-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.743402] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F8-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224196406172002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224196406172003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224196406172003, v:0}}], last_dump_ts_=1755224191713269, force_dump=false) [2025-08-15 10:16:36.743424] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F8-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.743556] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7719-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.743644] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7719-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.743715] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7887-0-0] [lt=23] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.745244] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4E-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.745290] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F8-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.745400] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F8-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.745628] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7887-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.745696] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC080-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224196671209000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224196671209000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224196671209000, v:0}}], last_dump_ts_=1755224191713762, force_dump=false) [2025-08-15 10:16:36.745720] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC080-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.745830] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7887-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.747211] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.747310] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.754165] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDC9-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.754304] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF093-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.755654] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC080-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.755749] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC080-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.786160] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8881-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.788602] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A60-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.788711] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8881-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:36.788729] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8881-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.791091] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A60-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:36.791108] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A60-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:36.791732] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=29] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:36.792652] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=12] mview_mds_op(tx_count=0) [2025-08-15 10:16:36.792664] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=8] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=830) [2025-08-15 10:16:36.805580] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7677-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.808589] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7677-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.810479] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7677-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.810584] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7677-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:36.821776] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709637][T1013_DRService][T1013][YB420ACB0122-000633A341D55699-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1013, thread_idx=1) [2025-08-15 10:16:36.825446] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [709637][T1013_DRService][T1013][YB420ACB0122-000633A341D5569A-0-0] [lt=14] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2402, tenant_id=1013, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:36.827821] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [709637][T1013_DRService][T1013][YB420ACB0122-000633A341D5569A-0-0] [lt=16] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2352, tenant_id=1014, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:36.842497] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:36.843742] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224196841546, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:16:36.843765] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224201843764, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:16:36.851897] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D49-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.855818] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475910][T1008_TimerWK2_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:36.943251] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=17] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224196549174, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224194403500002, replayable_scn:1755224194403500002, readable_scn:1755224194403500002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:36.985676] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3A3-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:36.995324] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5914-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.996071] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9F7-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:36.996382] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [709633][T1013_BackupDat][T1013][YB420ACB0122-000633A33E8598BF-0-0] [lt=16] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:37.059978] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DA-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.062543] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DA-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:37.062558] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DA-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.085961] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A6-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.088525] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A6-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:37.088540] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A6-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.107827] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C590E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.115732] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6D-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.118312] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6D-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:37.118329] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6D-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.140011] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0EF-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.152387] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF090-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.156657] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A8-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.159670] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A8-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.161629] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A8-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.161720] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A8-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.180530] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F8-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:37.183012] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F8-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:37.189082] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D47-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.190570] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:16:37.203665] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9FC-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.217348] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727956][T1018_TimerWK0_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=28] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:37.271645] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619486][T1012_TimerWK1_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:37.304693] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91A-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:37.307219] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91A-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:37.308068] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD528F-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:37.310636] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD528F-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:37.331967] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487605][T1004_TimerWK2_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=5] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:37.333658] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B58A2-0-0] [lt=11] [RESTORE] restore scheduler quit [2025-08-15 10:16:37.334969] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B58A2-0-0] [lt=13] get targets(ret=0, jobs=[], exec_tenant_id=1007, sql=select * from __all_recover_table_job) [2025-08-15 10:16:37.336156] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B58A3-0-0] [lt=16] get targets(ret=0, jobs=[], exec_tenant_id=1007, sql=select * from __all_import_table_job) [2025-08-15 10:16:37.336180] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B58A4-0-0] [lt=14][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224197336175, ref_timestamp=1755224137331587, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:16:37.336264] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B58A5-0-0] [lt=12] [RESTORE] restore scheduler start [2025-08-15 10:16:37.336274] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B58A6-0-0] [lt=7] [RESTORE] try process restore job [2025-08-15 10:16:37.337408] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B58A6-0-0] [lt=3] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:16:37.346576] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:37.347785] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224197346507, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:16:37.347809] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224202347807, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:16:37.395969] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C5-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:37.397203] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD532E-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:37.399733] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C5-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:37.399740] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD532E-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:37.438146] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:37.439366] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224197438073, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:16:37.439388] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224202439387, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:16:37.445090] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68A6-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.452528] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E65-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224197452193000, start_id=1755224197452193000, end_id=1755224197452193001) [2025-08-15 10:16:37.453186] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E65-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224197452193000, v:0}, tenant_id=1017) [2025-08-15 10:16:37.463441] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:37.463454] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:37.463460] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:37.465282] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD4-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:37.467804] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD4-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:37.478261] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D6C-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}) [2025-08-15 10:16:37.478299] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98D6C-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:16:37.504303] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D8-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:37.504825] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0ED-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.504870] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E75C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.506780] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D8-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:37.535466] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:37.535484] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:37.535494] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:37.536039] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:37.536051] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:37.536057] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:37.537369] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:37.541451] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:37.541463] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:37.541468] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:37.542747] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:37.547486] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:37.547498] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:37.547503] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:37.548766] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:37.552957] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:37.552970] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:37.552975] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:37.587245] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E6E-0-0] [lt=16] get number(ret=0, service_type_=0, range=1, base_id=1755224197586675000, start_id=1755224197586675000, end_id=1755224197586675001) [2025-08-15 10:16:37.587882] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E6E-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224197586675000, v:0}, tenant_id=1015) [2025-08-15 10:16:37.588018] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA670-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224197587730000, start_id=1755224197587730000, end_id=1755224197587730001) [2025-08-15 10:16:37.588662] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA670-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224197587730000, v:0}, tenant_id=1005) [2025-08-15 10:16:37.596396] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9F-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.597802] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A47-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.598977] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9F-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:37.598994] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF9F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.600392] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A47-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:37.600410] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A47-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.694018] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68A9-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.705226] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDBB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.709720] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:37.709733] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:37.709745] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:37.744671] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.746317] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F9-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.746912] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7888-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.747709] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771A-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.748253] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.749236] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F9-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.749678] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.749767] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.749859] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7888-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224197408076005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224197408076005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224197408076005, v:0}}], last_dump_ts_=1755224192719435, force_dump=false) [2025-08-15 10:16:37.749878] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7888-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.751145] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4F-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.751146] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.751341] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413F9-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.751815] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7888-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.751901] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7888-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.753175] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.753255] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C4F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.754240] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDCA-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.754378] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF094-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.756721] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC081-0-0] [lt=10] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224197408076003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224197408076003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224197408076003, v:0}}], last_dump_ts_=1755224192716835, force_dump=false) [2025-08-15 10:16:37.756743] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC081-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.759566] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC081-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.761485] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC081-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.761574] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC081-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.769020] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [728061][T1017_ArchiveSv][T1017][YB420ACB0122-000633A355A44E13-0-0] [lt=8] start do ObArchiveSchedulerService round(round=101448) [2025-08-15 10:16:37.772576] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [728061][T1017_ArchiveSv][T1017][YB420ACB0122-000633A355A44E13-0-0] [lt=10][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:16:37.772592] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [728061][T1017_ArchiveSv][T1017][YB420ACB0122-000633A355A44E13-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1018 AND dest_no = 0 AND name = 'state', exec_tenant_id=1017) [2025-08-15 10:16:37.772635] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [728061][T1017_ArchiveSv][T1017][YB420ACB0122-000633A355A44E13-0-0] [lt=8][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1018, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:16:37.772653] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [728061][T1017_ArchiveSv][T1017][YB420ACB0122-000633A355A44E13-0-0] [lt=14][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1018, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:16:37.772664] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [728061][T1017_ArchiveSv][T1017][YB420ACB0122-000633A355A44E13-0-0] [lt=8][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1018, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:16:37.791412] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8882-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.793718] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A61-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.793905] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8882-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:37.793922] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8882-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.796214] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A61-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:37.796231] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A61-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:37.811509] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:37.811603] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7678-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.812740] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224197811435, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:16:37.812761] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224202812760, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:16:37.813951] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224197812622, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:16:37.813970] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224202813969, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:16:37.814610] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7678-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.816418] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36CE-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224197815933000, start_id=1755224197815933000, end_id=1755224197815933001) [2025-08-15 10:16:37.816570] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7678-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.816646] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7678-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:37.817064] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36CE-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224197815933000, v:0}, tenant_id=1011) [2025-08-15 10:16:37.851947] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D4A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.857241] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC658-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.934904] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB6-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224197934762000, start_id=1755224197934762000, end_id=1755224197934762001) [2025-08-15 10:16:37.935565] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB6-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224197934762000, v:0}, tenant_id=1013) [2025-08-15 10:16:37.974862] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:37.976083] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224197974789, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:16:37.976105] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224202976104, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:16:37.987075] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3A4-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:37.995399] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5915-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:37.996141] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9F8-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.060451] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:38.062158] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224198060379, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:16:38.062190] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224203062188, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:16:38.064686] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DB-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.067296] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DB-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:38.067315] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DB-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.091319] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A7-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.093871] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A7-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:38.093890] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A7-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.107901] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C590F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.115359] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B6-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224198114743000, start_id=1755224198114743000, end_id=1755224198114743001) [2025-08-15 10:16:38.116184] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B6-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224198114743000, v:0}, tenant_id=1012) [2025-08-15 10:16:38.121024] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6E-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.123531] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6E-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:38.123548] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.134658] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463BE-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224198133642000, start_id=1755224198133642000, end_id=1755224198133642001) [2025-08-15 10:16:38.135491] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463BE-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224198133642000, v:0}, tenant_id=1) [2025-08-15 10:16:38.140083] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0F0-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.152460] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF091-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.162746] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.165760] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A9-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.167658] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A9-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.167738] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83A9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.188326] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F9-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:38.189157] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D48-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.190820] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38F9-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:38.203737] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9FD-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.290004] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95CD0-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}) [2025-08-15 10:16:38.290046] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95CD0-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:16:38.312675] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91B-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:38.313859] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91B-0-0] [lt=12] log restore source is empty, just skip(ret=-4018, user_tenant_id=1012) [2025-08-15 10:16:38.316377] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91B-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:38.316654] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5290-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:38.319173] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5290-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:38.405127] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C6-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:38.405127] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD532F-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:38.407622] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD532F-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:38.407621] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:38.445163] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68A7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.463338] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224196906764003, replayable_scn:1755224196906764003, readable_scn:1755224196906764003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224198460748, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224196906764003, replayable_scn:1755224196906764003, readable_scn:1755224196906764003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224198460748, ora_rowscn_=1755224198164212000, content_changed=true) [2025-08-15 10:16:38.463521] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:38.463539] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:38.463547] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:38.473689] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD5-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:38.476167] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD5-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:38.504905] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0EE-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.504946] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E75D-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.512152] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D9-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:38.514671] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62D9-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:38.514824] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF232-0-0] [lt=16] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.535505] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:38.535516] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:38.535521] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:38.536064] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224196406172003, replayable_scn:1755224196406172003, readable_scn:1755224196406172003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224198533382, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224196406172003, replayable_scn:1755224196406172003, readable_scn:1755224196406172003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224198533382, ora_rowscn_=1755224197747627000, content_changed=true) [2025-08-15 10:16:38.536245] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:38.536258] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:38.536267] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:38.541381] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224196407227002, replayable_scn:1755224196407227002, readable_scn:1755224196407227002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224198538640, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224196407227002, replayable_scn:1755224196407227002, readable_scn:1755224196407227002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224198538640, ora_rowscn_=1755224197746573000, content_changed=true) [2025-08-15 10:16:38.541557] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:38.541568] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:38.541575] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:38.545906] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1395-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84960938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.85351562GB}) [2025-08-15 10:16:38.545951] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1395-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:16:38.547394] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224196406172002, replayable_scn:1755224196406172002, readable_scn:1755224196406172002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224198543910, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224196406172002, replayable_scn:1755224196406172002, readable_scn:1755224196406172002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224198543910, ora_rowscn_=1755224197749735000, content_changed=true) [2025-08-15 10:16:38.547556] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:38.547566] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:38.547573] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:38.552812] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224196671209000, replayable_scn:1755224196671209000, readable_scn:1755224196671209000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224198550199, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224196671209000, replayable_scn:1755224196671209000, readable_scn:1755224196671209000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224198550199, ora_rowscn_=1755224197758166000, content_changed=true) [2025-08-15 10:16:38.552985] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:38.553001] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:38.553009] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:38.601781] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA0-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.603161] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A48-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.604340] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA0-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:38.604359] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA0-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.605725] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A48-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:38.605742] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A48-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.618575] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:38.619819] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224198618502, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:16:38.619840] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224203619839, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:16:38.684038] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [727353][T1015_MergeSche][T1015][YB420ACB0122-000633A34F52D649-0-0] [lt=12] zone_merge_mgr is already loaded(tenant_id=1015, global_merge_info={tenant_id:1015, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401710858, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194774854995, need_update:false}}, zone_merge_infos=[{tenant_id:1015, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194774855621, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401785541, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1015, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194774857517, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401792195, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1015, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194774859285, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401795570, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401680372000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:16:38.694092] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68AA-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.705302] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDBC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.709630] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224196406172003, replayable_scn:1755224196406172003, readable_scn:1755224196406172003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224198706887, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224196406172003, replayable_scn:1755224196406172003, readable_scn:1755224196406172003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224198706887, ora_rowscn_=1755224197748681000, content_changed=true) [2025-08-15 10:16:38.709812] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:38.709829] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:38.709840] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:38.713616] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF4-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1, thread_idx=0) [2025-08-15 10:16:38.714891] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-08-15 10:16:38.716106] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:38.729844] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.731004] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.732149] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.732171] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:38.732189] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.732201] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:38.732303] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=79] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-08-15 10:16:38.732311] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.732324] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.732336] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.732345] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:38.732423] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.732438] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.732447] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.732452] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.732461] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3772, tenant_id=1, ls_id={id:1}) [2025-08-15 10:16:38.732469] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:38.733661] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18770, tenant_id=1, acc_dr_task=0) [2025-08-15 10:16:38.750838] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771B-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224198409225002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224198409225002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224198409225002, v:0}}], last_dump_ts_=1755224193719321, force_dump=false) [2025-08-15 10:16:38.750864] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771B-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.752301] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FA-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.752887] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7889-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.753930] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771B-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224198409225002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224198409225001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224198409225002, v:0}}], last_dump_ts_=1755224193722533, force_dump=false) [2025-08-15 10:16:38.753960] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771B-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.754221] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C50-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.754317] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDCB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.754451] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF095-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.755203] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FA-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.755847] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771B-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.755972] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7889-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.756029] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.757177] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FA-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.757257] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FA-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.757273] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C50-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.757895] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7889-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.758036] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7889-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.759277] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C50-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.759356] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C50-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.762443] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC082-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.765274] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC082-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.767161] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC082-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.767273] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC082-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.770469] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5E-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224198769949000, start_id=1755224198769949000, end_id=1755224198769949001) [2025-08-15 10:16:38.771118] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5E-0-0] [lt=21] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224198769949000, v:0}, tenant_id=1016) [2025-08-15 10:16:38.796603] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8883-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.798872] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A62-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.799116] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [438124][T1005_BackupDat][T1005][YB420ACB0122-0006338207177DA5-0-0] [lt=12] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:38.799177] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8883-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:38.799195] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8883-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.801426] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A62-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:38.801444] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A62-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:38.810948] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8275-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1005, thread_idx=0) [2025-08-15 10:16:38.812234] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1005, only_for_display=false) [2025-08-15 10:16:38.813481] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:38.817622] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7679-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.820632] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7679-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.822531] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7679-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.822624] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7679-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:38.827457] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.828643] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.829843] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.829867] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1005, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:38.829884] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.829896] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1005, ls_id:{id:1}, replicas:[{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:38.830007] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=86] build ls locality stat map(tenant_id=1005, ls_id={id:1}) [2025-08-15 10:16:38.830015] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.830029] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.830040] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.830053] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:38.830131] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.830145] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.830156] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.830162] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.830171] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3870, tenant_id=1005, ls_id={id:1}) [2025-08-15 10:16:38.830180] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:38.831304] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19070, tenant_id=1005, acc_dr_task=0) [2025-08-15 10:16:38.832544] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1006, only_for_display=false) [2025-08-15 10:16:38.833737] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:38.847508] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.848658] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.849816] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.849837] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:38.849852] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.849862] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1}, replicas:[{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:38.849976] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=89] build ls locality stat map(tenant_id=1006, ls_id={id:1}) [2025-08-15 10:16:38.849983] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.849995] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.850004] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.850013] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:38.850087] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.850100] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.850108] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.850115] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.850122] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3752, tenant_id=1006, ls_id={id:1}) [2025-08-15 10:16:38.850130] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:38.851203] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:38.852019] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D4B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.872041] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.873197] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.874339] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.874361] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:38.874377] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.874389] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1001}, replicas:[{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:38.874490] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=78] build ls locality stat map(tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:16:38.874500] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.874511] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.874519] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.874528] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:38.874602] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.874614] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.874621] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.874628] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.874635] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3750, tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:16:38.874643] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:38.875712] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:38.896736] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.897886] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.899076] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.899097] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:38.899109] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=6] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.899119] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1002}, replicas:[{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:38.899228] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=85] build ls locality stat map(tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:16:38.899236] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.899247] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.899255] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.899264] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:38.899341] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.899353] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.899360] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.899369] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.899378] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3811, tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:16:38.899387] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:38.900545] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:38.920693] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619613][T1011_DRService][T1011][YB420ACB0122-0006338CC425E902-0-0] [lt=16] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1011, thread_idx=1) [2025-08-15 10:16:38.921911] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.923068] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.924312] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [2619613][T1011_DRService][T1011][YB420ACB0122-0006338CC425E903-0-0] [lt=17] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2344, tenant_id=1011, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:38.924368] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:38.924399] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:38.924424] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=15] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.924441] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=12] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1003}, replicas:[{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:38.924625] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=145] build ls locality stat map(tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:16:38.924639] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.924660] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=16] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.924678] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=14] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:38.924696] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=13] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:38.924844] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=112] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.924865] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=12] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.924881] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=11] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.924895] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=11] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:38.924908] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=10] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=4127, tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:16:38.924924] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=12] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:38.926247] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=11] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94928, tenant_id=1006, acc_dr_task=0) [2025-08-15 10:16:38.926627] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [2619613][T1011_DRService][T1011][YB420ACB0122-0006338CC425E903-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2293, tenant_id=1012, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:38.988831] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3A5-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:38.995471] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5916-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:38.996214] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9F9-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.070033] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DC-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.072655] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DC-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:39.072673] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DC-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.096576] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A8-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.099159] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A8-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:39.099176] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A8-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.107976] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5910-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.126321] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6F-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.128867] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6F-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:39.128883] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB6F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.140157] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0F1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.152534] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF092-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.168734] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AA-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.171931] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AA-0-0] [lt=17] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224198909922001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224198909922001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224198909922001, v:0}}], last_dump_ts_=1755224194141209, force_dump=false) [2025-08-15 10:16:39.171968] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AA-0-0] [lt=27] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.173971] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AA-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.174110] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AA-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.186924] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA651-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224199186509000, start_id=1755224199186509000, end_id=1755224199186509001) [2025-08-15 10:16:39.187675] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA651-0-0] [lt=21] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224199186509000, v:0}, tenant_id=1006) [2025-08-15 10:16:39.189227] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D49-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.196180] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38FA-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:39.198695] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38FA-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:39.203809] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9FE-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.251617] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2AA-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224199250682000, start_id=1755224199250682000, end_id=1755224199250682001) [2025-08-15 10:16:39.252427] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2AA-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224199250682000, v:0}, tenant_id=1008) [2025-08-15 10:16:39.252449] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2AA-0-0] [lt=14] zone_merge_mgr is already loaded(tenant_id=1008, global_merge_info={tenant_id:1008, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402107469, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194714424606, need_update:false}}, zone_merge_infos=[{tenant_id:1008, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194714425145, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402139417, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1008, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194714426903, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402144931, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1008, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194714428639, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402152998, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:16:39.321689] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91C-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:39.324184] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91C-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:39.324413] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5291-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:39.326867] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5291-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:39.412901] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5330-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:39.412933] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C7-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:39.415401] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5330-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:39.415449] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C7-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:39.445239] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68A8-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.463538] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:39.463549] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:39.463554] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:39.464877] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:39.478700] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A0986-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}) [2025-08-15 10:16:39.478734] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A0986-0-0] [lt=21] [DDL] execute ddl like stmt(ret=0, cost=37, ddl_arg=NULL) [2025-08-15 10:16:39.481483] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD6-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:39.483961] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD6-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:39.504970] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0EF-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.505020] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E75E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.519925] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62DA-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:39.522411] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62DA-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:39.528794] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227799527586}) [2025-08-15 10:16:39.528816] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=17][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:39.528826] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:39.534832] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227799533606}) [2025-08-15 10:16:39.534847] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:39.534861] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:39.535445] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224197408076005, replayable_scn:1755224197408076005, readable_scn:1755224197408076005, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224199532428, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224197408076005, replayable_scn:1755224197408076005, readable_scn:1755224197408076005, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224199532428, ora_rowscn_=1755224198819384000, content_changed=true) [2025-08-15 10:16:39.535640] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:39.535652] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:39.535657] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:39.536122] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227799534873}) [2025-08-15 10:16:39.536139] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:39.536152] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:39.536178] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:39.536189] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:39.536197] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:39.536955] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:39.541576] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:39.541592] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:39.541599] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:39.547590] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:39.547606] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:39.547613] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:39.553119] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:39.553130] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:39.553135] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:39.553144] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1006, cost_time_us=183, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:39.554432] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:39.607138] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA1-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.608489] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A49-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.609705] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA1-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:39.609723] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA1-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.611078] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A49-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:39.611097] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A49-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.639069] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:39.640316] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224199638997, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:16:39.640340] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224204640339, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:16:39.694165] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68AB-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.705377] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDBD-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.709779] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:39.709789] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:39.709794] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:39.711071] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:39.744739] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5EA-0-0] [lt=17] get number(ret=0, service_type_=0, range=1, base_id=1755224199744012000, start_id=1755224199744012000, end_id=1755224199744012001) [2025-08-15 10:16:39.745463] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5EA-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224199744012000, v:0}, tenant_id=1003) [2025-08-15 10:16:39.754388] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDCC-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.754524] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF096-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.757032] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.758201] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FB-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.759001] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788A-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224199411396004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224199411396004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224199411396004, v:0}}], last_dump_ts_=1755224194728475, force_dump=false) [2025-08-15 10:16:39.759022] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788A-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.759993] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771C-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.760320] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C51-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224199411396003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224199411396003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224199411396003, v:0}}], last_dump_ts_=1755224194729018, force_dump=false) [2025-08-15 10:16:39.760345] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C51-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.761029] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FB-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.761967] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.762088] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771C-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.762093] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.762918] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FB-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.763050] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FB-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.763730] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C51-0-0] [lt=18] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224199411396003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224199411396003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224199411396003, v:0}}], last_dump_ts_=1755224194732710, force_dump=false) [2025-08-15 10:16:39.763758] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C51-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.764081] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.764182] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.765728] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C51-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.765842] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C51-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.768218] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC083-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.771506] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC083-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.773514] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC083-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.773683] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC083-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.784744] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E67-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1003, thread_idx=0) [2025-08-15 10:16:39.785979] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1003, only_for_display=false) [2025-08-15 10:16:39.787182] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:39.800788] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.801944] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.801926] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8884-0-0] [lt=37] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.803078] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.803101] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1003, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:39.803117] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.803128] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1003, ls_id:{id:1}, replicas:[{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:39.803241] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=88] build ls locality stat map(tenant_id=1003, ls_id={id:1}) [2025-08-15 10:16:39.803249] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.803262] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.803273] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.803285] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:39.803363] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.803377] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.803386] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.803391] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.803396] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3751, tenant_id=1003, ls_id={id:1}) [2025-08-15 10:16:39.803406] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:39.804134] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A63-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.804460] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18482, tenant_id=1003, acc_dr_task=0) [2025-08-15 10:16:39.804461] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8884-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:39.804474] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8884-0-0] [lt=10] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.805666] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1004, only_for_display=false) [2025-08-15 10:16:39.806683] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A63-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:39.806701] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A63-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:39.806853] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:39.820587] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.821728] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.822878] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.822904] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:39.822920] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.822932] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:39.823045] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=88] build ls locality stat map(tenant_id=1004, ls_id={id:1}) [2025-08-15 10:16:39.823053] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.823066] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.823077] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.823089] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:39.823166] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.823180] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.823189] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.823194] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.823202] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3770, tenant_id=1004, ls_id={id:1}) [2025-08-15 10:16:39.823209] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:39.823652] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767A-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224199450339002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224199450339003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224199450339003, v:0}}], last_dump_ts_=1755224194793626, force_dump=false) [2025-08-15 10:16:39.823685] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767A-0-0] [lt=23] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.824275] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:39.826693] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767A-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224199450339002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224199450339003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224199450339003, v:0}}], last_dump_ts_=1755224194796562, force_dump=false) [2025-08-15 10:16:39.826723] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767A-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.828736] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.828825] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:39.844973] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.846115] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.847257] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.847279] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:39.847295] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.847306] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:39.847415] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=85] build ls locality stat map(tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:16:39.847427] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.847440] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.847447] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.847458] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:39.847535] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.847549] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.847559] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.847566] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.847575] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3756, tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:16:39.847585] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:39.848658] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:39.852075] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D4C-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.862040] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC659-0-0] [lt=12] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.869246] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.870391] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.871575] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.871598] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:39.871616] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.871626] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1002}, replicas:[{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:39.871736] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=86] build ls locality stat map(tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:16:39.871752] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=11] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.871765] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.871776] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.871787] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:39.871867] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.871881] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.871891] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.871895] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.871904] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3822, tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:16:39.871915] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:39.873002] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:39.887155] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C7-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224199887014000, start_id=1755224199887014000, end_id=1755224199887014001) [2025-08-15 10:16:39.888031] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C7-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224199887014000, v:0}, tenant_id=1004) [2025-08-15 10:16:39.893652] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.894797] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.895947] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:39.895969] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:39.895985] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.895997] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1003}, replicas:[{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:39.896102] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=82] build ls locality stat map(tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:16:39.896110] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.896122] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.896132] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:39.896143] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:39.896220] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.896234] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.896244] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.896250] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:39.896258] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3749, tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:16:39.896267] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:39.897339] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=92864, tenant_id=1004, acc_dr_task=0) [2025-08-15 10:16:39.944844] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224199550330, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224197408076003, replayable_scn:1755224197408076003, readable_scn:1755224197408076003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:39.990235] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3A6-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:39.995545] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5917-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:39.996287] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9FA-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.015849] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:40.074973] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DD-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.077542] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DD-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:40.077554] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DD-0-0] [lt=9] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.101893] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A9-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.104438] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A9-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:40.104453] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88A9-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.108052] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5911-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.119616] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C0-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1017, thread_idx=0) [2025-08-15 10:16:40.120863] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1017, only_for_display=false) [2025-08-15 10:16:40.122092] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=16] start try ls disaster recovery(ls_status_info={tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:40.131658] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB70-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.134208] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB70-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:40.134224] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB70-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.135924] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.137091] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=25] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.138233] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.138256] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1017, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:40.138273] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.138284] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1017, ls_id:{id:1}, replicas:[{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:40.138386] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=80] build ls locality stat map(tenant_id=1017, ls_id={id:1}) [2025-08-15 10:16:40.138399] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.138412] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.138418] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.138430] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:40.138507] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.138521] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.138531] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.138536] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.138544] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3857, tenant_id=1017, ls_id={id:1}) [2025-08-15 10:16:40.138556] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=10] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:40.139680] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18817, tenant_id=1017, acc_dr_task=0) [2025-08-15 10:16:40.140230] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0F2-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.140878] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1018, only_for_display=false) [2025-08-15 10:16:40.142081] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:40.152610] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF093-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.155809] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.156970] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.158113] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.158136] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:40.158153] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.158164] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1}, replicas:[{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:40.158267] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1}) [2025-08-15 10:16:40.158275] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.158288] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.158299] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.158309] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:40.158385] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.158399] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.158408] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.158413] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.158421] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3759, tenant_id=1018, ls_id={id:1}) [2025-08-15 10:16:40.158431] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:40.159490] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:40.175089] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AB-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224199911204002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224199911204002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224199911204002, v:0}}], last_dump_ts_=1755224195144341, force_dump=false) [2025-08-15 10:16:40.175113] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AB-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.178268] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AB-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.180323] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AB-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.180329] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.180419] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AB-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.181492] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.182645] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.182667] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:40.182684] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.182696] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1001}, replicas:[{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:40.182798] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:16:40.182806] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.182819] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.182830] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.182841] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:40.182917] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.182931] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.182947] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=12] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.182953] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.182961] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3795, tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:16:40.182967] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:40.184026] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:40.189300] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D4A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.203886] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9FF-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.203994] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38FB-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:40.204738] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.205895] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.206484] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38FB-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:40.207061] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.207083] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:40.207100] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.207111] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1002}, replicas:[{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:40.207212] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:16:40.207226] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.207239] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.207250] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.207261] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:40.207344] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.207358] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.207368] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.207373] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.207381] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3790, tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:16:40.207392] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:40.208490] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:40.216795] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C8-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224200216103000, start_id=1755224200216103000, end_id=1755224200216103001) [2025-08-15 10:16:40.217511] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C8-0-0] [lt=19] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224200216103000, v:0}, tenant_id=1007) [2025-08-15 10:16:40.218553] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727239][T1015_DRService][T1015][YB420ACB0122-000633A351E5562E-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1015, thread_idx=1) [2025-08-15 10:16:40.222192] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [727239][T1015_DRService][T1015][YB420ACB0122-000633A351E5562F-0-0] [lt=14] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2382, tenant_id=1015, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:40.224505] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [727239][T1015_DRService][T1015][YB420ACB0122-000633A351E5562F-0-0] [lt=17] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2290, tenant_id=1016, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:40.229297] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.230462] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.231648] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:40.231671] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:40.231688] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.231700] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1003}, replicas:[{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:40.231805] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=82] build ls locality stat map(tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:16:40.231812] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.231825] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.231839] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=11] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:40.231851] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:40.231928] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.231946] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.231956] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.231961] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:40.231970] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3818, tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:16:40.231979] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:40.233039] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93345, tenant_id=1018, acc_dr_task=0) [2025-08-15 10:16:40.290476] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D85-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}) [2025-08-15 10:16:40.290515] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D85-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:16:40.325527] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [438023][T1006_TimerWK2_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=74] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:40.325561] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [438023][T1006_TimerWK2_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=24] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:40.325577] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [438023][T1006_TimerWK2_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=12] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:40.326797] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [438021][T1006_TimerWK0_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=16] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:40.329744] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91D-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:40.332172] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5292-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:40.332246] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91D-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:40.334653] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5292-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:40.340871] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [17712][T1_BackupDataSr][T1][YB420ACB0122-0006337F03F7B811-0-0] [lt=15] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:40.420809] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C8-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:40.420809] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5331-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:40.423287] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C8-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:40.423294] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5331-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:40.445314] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68A9-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.463580] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:40.463593] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:40.463599] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:40.489226] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD7-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:40.491716] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD7-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:40.505044] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0F0-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.505093] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E75F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.515088] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF233-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.527887] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62DB-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:40.530407] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62DB-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:40.535666] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:40.535677] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:40.535682] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:40.536231] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:40.536243] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:40.536258] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:40.541989] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:40.542005] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:40.542015] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:40.546363] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D584E-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.85351562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.85742188GB}) [2025-08-15 10:16:40.546401] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D584E-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:16:40.547739] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:40.547752] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:40.547757] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:40.553153] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:40.553163] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:40.553168] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:40.604619] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17716][T1_DRService][T1][YB420ACB0122-0006337F0497B7A2-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1, thread_idx=1) [2025-08-15 10:16:40.608187] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [17716][T1_DRService][T1][YB420ACB0122-0006337F0497B7A3-0-0] [lt=11] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2316, tenant_id=1, need_clean_task=true, service_epoch_to_check=2) [2025-08-15 10:16:40.612448] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA2-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.613818] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4A-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.615097] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA2-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:40.615113] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA2-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.616443] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4A-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:40.616460] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4A-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.634191] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:40.635446] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224200634118, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:16:40.635467] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224205635466, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:16:40.694246] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68AC-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.705451] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDBE-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.709853] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:40.709865] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:40.709870] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:40.749496] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438021][T1006_TimerWK0_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:40.754461] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDCD-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.754597] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF097-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.763129] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.763995] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FC-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.765137] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788B-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.766272] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771D-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.766682] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C52-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.766953] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FC-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.768242] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.768244] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788B-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.768322] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.768874] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FC-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.768955] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FC-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.769731] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C52-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.770174] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.770252] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.771650] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C52-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.771735] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C52-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.774633] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC084-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.777805] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC084-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.779833] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC084-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.779933] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC084-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.807197] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8885-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.809401] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A64-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.809728] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8885-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:40.809745] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8885-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.811992] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A64-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:40.812010] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A64-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:40.829809] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767B-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.832907] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767B-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.834901] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.835047] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767B-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:40.852151] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D4D-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.958740] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:40.959960] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224200958669, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:16:40.959984] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224205959982, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:16:40.991623] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3A7-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:40.995620] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5918-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:40.996359] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9FB-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.079712] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DE-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.082243] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DE-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:41.082259] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DE-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.107124] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:41.107178] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AA-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.108126] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5912-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.108510] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224201098527, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:16:41.108538] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224206108536, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:16:41.108559] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224206108556, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:41.109749] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AA-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:41.109765] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AA-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.137000] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB71-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.139560] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB71-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:41.139577] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB71-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.140301] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0F3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.152688] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF094-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.181414] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AC-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.184416] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AC-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.186282] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AC-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.186373] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AC-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.189375] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D4B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.201621] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:41.202829] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224201201550, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:16:41.202851] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224206202849, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:16:41.203519] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:41.203951] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA00-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.204892] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224201203453, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:16:41.204916] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224206204915, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:41.211861] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38FC-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:41.215513] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38FC-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:41.337699] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91E-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:41.340035] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5293-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:41.340251] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91E-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:41.342539] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5293-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:41.428634] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5332-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:41.428648] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C9-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:41.430763] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E55-0-0] [lt=16] get number(ret=0, service_type_=0, range=1, base_id=1755224201429909000, start_id=1755224201429909000, end_id=1755224201429909001) [2025-08-15 10:16:41.431131] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5332-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:41.431155] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86C9-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:41.431568] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E55-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224201429909000, v:0}, tenant_id=1018) [2025-08-15 10:16:41.445389] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68AA-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.463430] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224199911204002, replayable_scn:1755224199911204002, readable_scn:1755224199911204002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224201460420, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224199911204002, replayable_scn:1755224199911204002, readable_scn:1755224199911204002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224201460420, ora_rowscn_=1755224201183575000, content_changed=true) [2025-08-15 10:16:41.463593] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:41.463604] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:41.463611] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:41.479145] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A0A6D-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}) [2025-08-15 10:16:41.479183] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A0A6D-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:16:41.497025] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD8-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:41.499489] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD8-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:41.505118] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0F1-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.505165] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E760-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.535730] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62DC-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:41.536176] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224199411396004, replayable_scn:1755224199411396004, readable_scn:1755224199411396004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224201533015, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224199411396004, replayable_scn:1755224199411396004, readable_scn:1755224199411396004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224201533015, ora_rowscn_=1755224200766121000, content_changed=true) [2025-08-15 10:16:41.536362] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:41.536374] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:41.536379] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:41.536727] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:41.536736] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:41.536744] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:41.538287] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62DC-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:41.541560] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224199411396002, replayable_scn:1755224199411396002, readable_scn:1755224199411396002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224201539328, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224199411396002, replayable_scn:1755224199411396002, readable_scn:1755224199411396002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224201539328, ora_rowscn_=1755224200765091000, content_changed=true) [2025-08-15 10:16:41.541741] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:41.541751] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:41.541761] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:41.547647] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224199411396003, replayable_scn:1755224199411396003, readable_scn:1755224199411396003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224201544595, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224199411396003, replayable_scn:1755224199411396003, readable_scn:1755224199411396003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224201544595, ora_rowscn_=1755224200768182000, content_changed=true) [2025-08-15 10:16:41.547819] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:41.547829] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:41.547837] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:41.553048] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224199411396003, replayable_scn:1755224199411396003, readable_scn:1755224199411396003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224201549863, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224199411396003, replayable_scn:1755224199411396003, readable_scn:1755224199411396003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224201549863, ora_rowscn_=1755224200776571000, content_changed=true) [2025-08-15 10:16:41.553224] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:41.553233] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:41.553240] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:41.617831] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA3-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.619102] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4B-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.620426] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA3-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:41.620444] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA3-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.621671] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4B-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:41.621690] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.694300] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68AD-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.705521] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDBF-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.709812] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224199411396004, replayable_scn:1755224199411396004, readable_scn:1755224199411396004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224201706607, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224199411396004, replayable_scn:1755224199411396004, readable_scn:1755224199411396004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224201706607, ora_rowscn_=1755224200767128000, content_changed=true) [2025-08-15 10:16:41.710010] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:41.710027] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:41.710041] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:41.754537] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDCE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.754673] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF098-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.769342] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.769897] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FD-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224201414128004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224201414128004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224201414128004, v:0}}], last_dump_ts_=1755224196740443, force_dump=false) [2025-08-15 10:16:41.769925] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FD-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.771194] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.772417] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771E-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.772687] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C53-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.773003] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FD-0-0] [lt=15] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224201414128004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224201414128004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224201414128004, v:0}}], last_dump_ts_=1755224196743401, force_dump=false) [2025-08-15 10:16:41.773028] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FD-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.774183] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788C-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.774471] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.774680] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.774962] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FD-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.775078] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FD-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.775705] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C53-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.776207] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.776297] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.777617] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C53-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.777706] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C53-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.780824] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC085-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.783726] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC085-0-0] [lt=15] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224201675071000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224201675071001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224201675071001, v:0}}], last_dump_ts_=1755224196745695, force_dump=false) [2025-08-15 10:16:41.783749] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC085-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.785642] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC085-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.785730] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC085-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.793043] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709535][T1014_TimerWK2_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:41.812495] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8886-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.814654] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A65-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.815074] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8886-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:41.815093] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8886-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.817214] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A65-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:41.817232] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A65-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:41.824786] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA5-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224201824455000, start_id=1755224201824455000, end_id=1755224201824455001) [2025-08-15 10:16:41.825548] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA5-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224201824455000, v:0}, tenant_id=1014) [2025-08-15 10:16:41.835980] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.838900] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767C-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.840762] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.840852] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:41.843836] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:41.845070] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224201843764, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:16:41.845094] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224206845092, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:16:41.852226] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D4E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.861221] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475910][T1008_TimerWK2_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:41.866980] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC65A-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.993030] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3A8-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:41.995691] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5919-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:41.996413] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9FC-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.072733] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D7-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1015, thread_idx=0) [2025-08-15 10:16:42.074003] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1015, only_for_display=false) [2025-08-15 10:16:42.075219] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:42.084964] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DF-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.087444] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DF-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:42.087460] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95DF-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.089154] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.090341] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.091492] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.091513] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1015, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:42.091528] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.091537] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1015, ls_id:{id:1}, replicas:[{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:42.091643] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=83] build ls locality stat map(tenant_id=1015, ls_id={id:1}) [2025-08-15 10:16:42.091653] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.091663] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.091671] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.091680] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:42.091755] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=54] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.091767] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.091774] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.091781] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.091787] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3794, tenant_id=1015, ls_id={id:1}) [2025-08-15 10:16:42.091795] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:42.092866] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18864, tenant_id=1015, acc_dr_task=0) [2025-08-15 10:16:42.094143] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1016, only_for_display=false) [2025-08-15 10:16:42.095342] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:42.108199] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5913-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.109366] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.110523] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.111683] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.111707] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:42.111719] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.111730] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1}, replicas:[{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:42.111833] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=80] build ls locality stat map(tenant_id=1016, ls_id={id:1}) [2025-08-15 10:16:42.111840] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.111853] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.111865] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.111875] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:42.111959] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.111974] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.111984] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.111989] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.111998] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3784, tenant_id=1016, ls_id={id:1}) [2025-08-15 10:16:42.112004] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:42.112475] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AB-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.113102] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:42.115127] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AB-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:42.115148] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AB-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.134061] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.135221] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.136363] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.136384] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:42.136404] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.136416] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1001}, replicas:[{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:42.136526] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=87] build ls locality stat map(tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:16:42.136534] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.136547] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.136558] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.136581] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=18] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:42.136657] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.136671] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.136680] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.136690] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.136696] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3775, tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:16:42.136706] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:42.137870] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:42.140376] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0F4-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.142356] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB72-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.144904] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB72-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:42.144923] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB72-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.152756] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF095-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.158780] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.159931] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.161098] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.161124] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:42.161143] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.161157] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1002}, replicas:[{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:42.161333] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=139] build ls locality stat map(tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:16:42.161349] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.161365] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=12] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.161378] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=10] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.161391] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:42.161526] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=101] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.161539] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.161550] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.161561] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.161570] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3950, tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:16:42.161593] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=18] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:42.162754] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:42.183717] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.184883] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.186062] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:42.186083] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:42.186098] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.186106] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1003}, replicas:[{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:42.186209] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=80] build ls locality stat map(tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:16:42.186216] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.186227] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.186232] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=4] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:42.186241] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:42.186319] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.186330] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.186337] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=4] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.186344] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:42.186350] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3804, tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:16:42.186358] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:42.187337] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AD-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.187439] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94560, tenant_id=1016, acc_dr_task=0) [2025-08-15 10:16:42.189448] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D4C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.190443] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AD-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.192361] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AD-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.192461] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AD-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.195506] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=16] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:42.204020] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA01-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.218681] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:42.220911] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38FD-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:42.223408] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38FD-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:42.272999] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619486][T1012_TimerWK1_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:42.290944] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B63A9-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}) [2025-08-15 10:16:42.290984] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B63A9-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:16:42.333298] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=68] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:42.343485] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC6-0-0] [lt=7] start do ObArchiveSchedulerService round(round=195067) [2025-08-15 10:16:42.345611] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91F-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:42.345836] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC6-0-0] [lt=15][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:16:42.345851] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC6-0-0] [lt=11][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1 AND dest_no = 0 AND name = 'state', exec_tenant_id=1) [2025-08-15 10:16:42.345905] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC6-0-0] [lt=8][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:16:42.345919] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC6-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:16:42.345931] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC6-0-0] [lt=9][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:16:42.347878] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:42.347909] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5294-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:42.348167] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E91F-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:42.349135] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224202347807, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:16:42.349158] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224207349156, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:16:42.350446] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5294-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:42.436429] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86CA-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:42.436526] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5333-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:42.438989] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86CA-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:42.439464] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:42.439951] INFO [RS] finish (ob_mview_maintenance_task.cpp:280) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=12] mview maintenace task finish(tenant_id_=1008, round_=105551, status_=100, error_code_=0, cost_us_=20001375, prepare_cost_us_=0, gc_mview_cost_us_=1220, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:16:42.440169] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5333-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:42.440655] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224202439387, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:16:42.440678] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224207440676, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:16:42.445464] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68AB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.463673] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:42.463685] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:42.463691] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:42.489942] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728063][T1017_DRService][T1017][YB420ACB0122-000633A358855628-0-0] [lt=15] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1017, thread_idx=1) [2025-08-15 10:16:42.494328] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [728063][T1017_DRService][T1017][YB420ACB0122-000633A358855629-0-0] [lt=23] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2511, tenant_id=1017, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:42.496694] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [728063][T1017_DRService][T1017][YB420ACB0122-000633A358855629-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2346, tenant_id=1018, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:42.504783] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD9-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:42.505192] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0F2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.505237] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E761-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.507339] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FD9-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:42.515369] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF234-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.535614] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224200592730003, replayable_scn:1755224200592730003, readable_scn:1755224200592730003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224202532485, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224200592730003, replayable_scn:1755224200592730003, readable_scn:1755224200592730003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224202532485, ora_rowscn_=1755224201838106000, content_changed=true) [2025-08-15 10:16:42.535803] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:42.535815] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:42.535827] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:42.536422] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:42.536435] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:42.536441] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:42.537811] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:42.540458] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227802539287}) [2025-08-15 10:16:42.540475] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:42.540488] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:42.541795] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:42.541805] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:42.541812] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:42.543104] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:42.544634] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62DD-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:42.546394] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227802545220}) [2025-08-15 10:16:42.546409] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:42.546422] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:42.546844] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1398-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.85742188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80273438GB}) [2025-08-15 10:16:42.546883] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1398-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:16:42.547607] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227802546436}) [2025-08-15 10:16:42.547621] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:42.547630] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:42.547688] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62DD-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:42.547867] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:42.547879] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:42.547887] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:42.549196] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:42.553334] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:42.553349] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:42.553361] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:42.553375] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1006, cost_time_us=215, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:42.623187] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA4-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.624323] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4C-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.625712] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA4-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:42.625732] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA4-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.626827] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4C-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:42.626846] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4C-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.694378] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68AE-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.705598] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDC0-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.710096] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:42.710110] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:42.710115] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:42.754612] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDCF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.754746] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF099-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.775699] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.775962] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FE-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.777164] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.778680] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C54-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.779000] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771F-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.779345] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FE-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.780111] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788D-0-0] [lt=20] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224202414708003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224202414708003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224202414708003, v:0}}], last_dump_ts_=1755224197749858, force_dump=false) [2025-08-15 10:16:42.780136] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788D-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.780978] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771F-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.781111] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D771F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.781352] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FE-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.781441] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FE-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.781652] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C54-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.782022] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.782134] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.783612] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C54-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.783719] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C54-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.786653] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC086-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224202414708003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224202414708004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224202414708004, v:0}}], last_dump_ts_=1755224197756720, force_dump=false) [2025-08-15 10:16:42.786674] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC086-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.789520] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC086-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.791389] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC086-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.791484] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC086-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.812832] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:42.814085] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224202812760, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:16:42.814108] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224207814106, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:16:42.815370] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224202813969, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:16:42.815391] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224207815390, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:16:42.817799] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8887-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.819888] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A66-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.820350] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8887-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:42.820368] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8887-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.822420] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A66-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:42.822439] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A66-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:42.841857] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767D-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.844962] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767D-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.847093] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.847187] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:42.852296] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D4F-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.946419] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224202550393, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224200411859003, replayable_scn:1755224200411859003, readable_scn:1755224200411859003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:42.976183] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:42.977417] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224202976104, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:16:42.977438] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224207977437, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:16:42.994418] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3A9-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:42.995752] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C591A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:42.996485] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9FD-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.062261] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:43.063495] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224203062188, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:16:43.063522] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224208063521, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:16:43.089645] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E0-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.092207] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E0-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:43.092225] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E0-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.108274] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5914-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.117888] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AC-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.120453] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AC-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:43.120473] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AC-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.140447] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0F5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.147748] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB73-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.150247] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB73-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:43.150264] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB73-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.152828] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF096-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.164017] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EE-0-0] [lt=16] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1013, thread_idx=0) [2025-08-15 10:16:43.165290] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1013, only_for_display=false) [2025-08-15 10:16:43.166552] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:43.180808] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.181988] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.183144] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.183167] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1013, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:43.183184] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.183196] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1013, ls_id:{id:1}, replicas:[{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:43.183298] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=79] build ls locality stat map(tenant_id=1013, ls_id={id:1}) [2025-08-15 10:16:43.183306] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.183320] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.183331] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.183343] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:43.183420] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.183434] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.183443] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.183449] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.183454] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3813, tenant_id=1013, ls_id={id:1}) [2025-08-15 10:16:43.183466] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:43.184530] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19240, tenant_id=1013, acc_dr_task=0) [2025-08-15 10:16:43.185762] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1014, only_for_display=false) [2025-08-15 10:16:43.186977] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:43.189518] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D4D-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.193454] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AE-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.196121] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:340) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AE-0-0] [lt=14] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1014, ls_id:{id:1}, sync_scn:{val:1755224202916319003, v:0}, readable_scn:{val:1755224202916319003, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-08-15 10:16:43.196562] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AE-0-0] [lt=27] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.198439] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AE-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.198550] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AE-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.198585] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:459) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AE-0-0] [lt=5] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1755224200913322002, v:0}) [2025-08-15 10:16:43.200855] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.202108] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.203304] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.203328] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:43.203346] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.203358] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1}, replicas:[{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:43.203461] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=81] build ls locality stat map(tenant_id=1014, ls_id={id:1}) [2025-08-15 10:16:43.203471] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.203484] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.203495] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.203502] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:43.203579] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.203595] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.203604] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.203610] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.203618] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3912, tenant_id=1014, ls_id={id:1}) [2025-08-15 10:16:43.203629] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:43.204093] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA02-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.204693] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:43.225844] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.227031] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.228204] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.228227] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:43.228245] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.228256] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1001}, replicas:[{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:43.228361] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=82] build ls locality stat map(tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:16:43.228370] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.228383] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.228393] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.228401] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:43.228479] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.228496] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.228505] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.228512] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.228520] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3864, tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:16:43.228529] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:43.228913] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38FE-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:43.229689] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:43.231425] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38FE-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:43.250754] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.251919] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.253075] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.253098] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:43.253116] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.253128] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1002}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:43.253238] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=86] build ls locality stat map(tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:16:43.253247] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.253260] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.253268] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.253276] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:43.253354] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.253369] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.253379] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.253384] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.253394] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3773, tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:16:43.253404] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:43.254485] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:43.275413] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.276566] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.277716] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:43.277738] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:43.277755] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.277766] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1003}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:43.277870] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=81] build ls locality stat map(tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:16:43.277878] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.277890] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.277900] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:43.277913] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:43.278003] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=66] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.278018] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.278028] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.278034] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:43.278042] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3790, tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:16:43.278052] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:43.279130] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94585, tenant_id=1014, acc_dr_task=0) [2025-08-15 10:16:43.353486] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E920-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:43.355806] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5295-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:43.356087] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E920-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:43.359406] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5295-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:43.444426] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86CB-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:43.445407] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5334-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:43.445537] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68AC-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.447004] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86CB-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:43.447928] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5334-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:43.463651] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:43.463669] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:43.463675] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:43.479614] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A0A7E-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}) [2025-08-15 10:16:43.479653] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A0A7E-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:16:43.505267] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0F3-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.505311] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E762-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.512654] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FDA-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:43.516309] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FDA-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:43.535814] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:43.535826] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:43.535832] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:43.536449] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:43.536459] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:43.536464] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:43.541781] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:43.541792] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:43.541797] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:43.547954] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:43.547967] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:43.547973] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:43.552922] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62DE-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:43.553310] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:43.553320] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:43.553327] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:43.555439] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62DE-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:43.619913] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:43.621150] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224203619839, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:16:43.621172] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224208621170, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:16:43.628479] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA5-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.629536] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4D-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.631097] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA5-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:43.631116] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA5-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.632080] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4D-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:43.632098] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4D-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.694455] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68AF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.705673] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDC1-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.710126] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:43.710141] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:43.710146] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:43.754686] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDD0-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.754816] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF09A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.782377] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FF-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.782490] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7720-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224203417049003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224203417049003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224203417049003, v:0}}], last_dump_ts_=1755224198750837, force_dump=false) [2025-08-15 10:16:43.782514] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7720-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.783161] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.784637] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C55-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.785447] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FF-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.785530] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7720-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224203417049003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224203417049003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224203417049003, v:0}}], last_dump_ts_=1755224198753929, force_dump=false) [2025-08-15 10:16:43.785552] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7720-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.786203] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788E-0-0] [lt=23] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.787367] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FF-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.787494] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413FF-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.787494] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7720-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.787593] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7720-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.787651] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C55-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.788140] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788E-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.788219] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.789575] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C55-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.789663] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C55-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.792441] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC087-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.795307] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC087-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.797162] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC087-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.797240] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC087-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.823139] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8888-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.825087] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A67-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.825673] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8888-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:43.825691] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8888-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.827580] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A67-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:43.827598] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A67-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:43.848205] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.851146] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767E-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.852366] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D50-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.853156] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767E-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.853340] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:43.871901] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC65B-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.995826] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C591B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:43.995870] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3AA-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:43.996560] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9FE-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.094395] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E1-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.096960] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E1-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:44.096979] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E1-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.108347] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5915-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.123197] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AD-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.125779] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AD-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:44.125798] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AD-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.140522] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0F6-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.152899] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF097-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.153012] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB74-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.155544] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB74-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:44.155563] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB74-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.189594] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D4E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.199564] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AF-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.202693] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AF-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224203917904001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224203917904002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224203917904002, v:0}}], last_dump_ts_=1755224199171931, force_dump=false) [2025-08-15 10:16:44.202719] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AF-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.204166] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA03-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.204699] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AF-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.204796] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83AF-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.236860] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38FF-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:44.239358] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38FF-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:44.291469] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95CE0-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}) [2025-08-15 10:16:44.291508] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95CE0-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:16:44.361432] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E921-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:44.363922] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E921-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:44.364651] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5296-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:44.367132] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5296-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:44.445613] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68AD-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.452420] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86CC-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:44.453199] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5335-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:44.455004] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86CC-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:44.456951] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5335-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:44.463619] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224202916319003, replayable_scn:1755224202916319003, readable_scn:1755224202916319003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224204460439, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224202916319003, replayable_scn:1755224202916319003, readable_scn:1755224202916319003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224204460439, ora_rowscn_=1755224204201771000, content_changed=true) [2025-08-15 10:16:44.463809] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:44.463819] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:44.463826] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:44.465138] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:44.505338] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0F4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.505384] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E763-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.515631] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF235-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.521663] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FDB-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:44.524184] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FDB-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:44.535852] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:44.535862] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:44.535868] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:44.536338] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224202414708004, replayable_scn:1755224202414708004, readable_scn:1755224202414708004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224204532979, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224202414708004, replayable_scn:1755224202414708004, readable_scn:1755224202414708004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224204532979, ora_rowscn_=1755224203784281000, content_changed=true) [2025-08-15 10:16:44.536504] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:44.536514] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:44.536522] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:44.537150] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:44.541713] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224202415761003, replayable_scn:1755224202415761003, readable_scn:1755224202415761003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224204539299, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224202415761003, replayable_scn:1755224202415761003, readable_scn:1755224202415761003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224204539299, ora_rowscn_=1755224203784281000, content_changed=true) [2025-08-15 10:16:44.541893] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:44.541905] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:44.541911] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:44.547385] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5851-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80273438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}) [2025-08-15 10:16:44.547421] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5851-0-0] [lt=21] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:16:44.547835] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224202414708004, replayable_scn:1755224202414708004, readable_scn:1755224202414708004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224204544567, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224202414708004, replayable_scn:1755224202414708004, readable_scn:1755224202414708004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224204544567, ora_rowscn_=1755224203786388000, content_changed=true) [2025-08-15 10:16:44.548030] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:44.548042] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:44.548047] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:44.553226] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224202414708004, replayable_scn:1755224202414708004, readable_scn:1755224202414708004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224204549834, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224202414708004, replayable_scn:1755224202414708004, readable_scn:1755224202414708004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224204549834, ora_rowscn_=1755224203793761000, content_changed=true) [2025-08-15 10:16:44.553395] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:44.553412] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:44.553420] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:44.554809] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:44.560786] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62DF-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:44.564453] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62DF-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:44.633877] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA6-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.634098] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:104) [17458][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=15] start checking lost replicas [2025-08-15 10:16:44.634811] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4E-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.636454] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA6-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:44.636473] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA6-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.637316] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4E-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:44.637334] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4E-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.640415] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:44.641620] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224204640339, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:16:44.641642] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224209641640, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:16:44.694533] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68B0-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.705753] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDC2-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.710005] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224202414708003, replayable_scn:1755224202414708003, readable_scn:1755224202414708003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224204706499, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224202414708003, replayable_scn:1755224202414708003, readable_scn:1755224202414708003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224204706499, ora_rowscn_=1755224203785334000, content_changed=true) [2025-08-15 10:16:44.710195] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:44.710205] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:44.710212] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:44.711530] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:44.736064] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476036][T1007_DRService][T1007][YB420ACB0122-0006338215870D10-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1007, thread_idx=1) [2025-08-15 10:16:44.740336] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [476036][T1007_DRService][T1007][YB420ACB0122-0006338215870D11-0-0] [lt=12] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2564, tenant_id=1007, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:44.742652] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [476036][T1007_DRService][T1007][YB420ACB0122-0006338215870D11-0-0] [lt=14] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2296, tenant_id=1008, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:44.754760] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDD1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.754893] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF09B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.788464] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841400-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.788792] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7721-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.789180] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788F-0-0] [lt=10] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224204418420002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224204764365000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224204764365000, v:0}}], last_dump_ts_=1755224199759000, force_dump=false) [2025-08-15 10:16:44.789206] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788F-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.790936] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C56-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224204418420003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224204418420004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224204418420004, v:0}}], last_dump_ts_=1755224199760319, force_dump=false) [2025-08-15 10:16:44.790973] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C56-0-0] [lt=27] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.791582] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841400-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.792282] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788F-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.792413] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7721-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.793588] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841400-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.793683] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841400-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.794046] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C56-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224204418420003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224204418420004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224204418420004, v:0}}], last_dump_ts_=1755224199763729, force_dump=false) [2025-08-15 10:16:44.794074] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C56-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.794259] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.794350] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB788F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.794969] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7721-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.795078] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7721-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.796211] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C56-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.796308] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C56-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.798185] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC088-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.801253] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC088-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.803105] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC088-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.803200] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC088-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.828480] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8889-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.830284] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A68-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.831079] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8889-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:44.831095] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8889-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.832828] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A68-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:44.832847] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A68-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:44.852440] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D51-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.854512] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767F-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224204631838000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224204766472000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224204766472000, v:0}}], last_dump_ts_=1755224199823651, force_dump=false) [2025-08-15 10:16:44.854539] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767F-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.857555] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767F-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224204631838000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224204766472000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224204766472000, v:0}}], last_dump_ts_=1755224199826692, force_dump=false) [2025-08-15 10:16:44.857578] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767F-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.859479] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767F-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.859555] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D767F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:44.995898] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C591C-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.996633] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9FF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:44.997344] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3AB-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:45.017127] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=8] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:45.099141] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E2-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.101720] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E2-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:45.101737] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E2-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.108423] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5916-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.128575] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AE-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.131150] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AE-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:45.131167] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AE-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.140595] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0F7-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.146119] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=19] start do ObArchiveSchedulerService round(round=168693) [2025-08-15 10:16:45.147454] INFO [ARCHIVE] checkpoint_ (ob_tenant_archive_scheduler.cpp:682) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=15] schedule archive(round_info={key:{tenant_id:1008, dest_no:0}, incarnation:1, dest_id:1001, round_id:1, state:{status:"DOING"}, start_scn:{val:1752696006021581000, v:0}, checkpoint_scn:{val:1755224040686907000, v:0}, max_scn:{val:1755224115293438002, v:0}, compatible:{version:1}, base_piece_id:1, used_piece_id:30, piece_switch_interval:86400000000, frozen_input_bytes:13323961132, frozen_output_bytes:13323961132, active_input_bytes:117537394, active_output_bytes:117537394, deleted_input_bytes:0, deleted_output_bytes:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com", comment:""}) [2025-08-15 10:16:45.148889] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476034][T1007_ArchiveSv][T1008][YB420ACB0122-0006338212A554C0-0-0] [lt=22] get number(ret=0, service_type_=0, range=1, base_id=1755224205148215000, start_id=1755224205148215000, end_id=1755224205148215001) [2025-08-15 10:16:45.148911] INFO [SHARE] get_backup_scn (ob_backup_struct.cpp:2155) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=16] get tenant gts(ret=0, ret="OB_SUCCESS", tenant_id=1008, scn={val:1755224205148215000, v:0}) [2025-08-15 10:16:45.148930] INFO [SHARE] count_ (ob_archive_checkpoint.cpp:173) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=11] print count result(ret=0, counter={ls_count:4, deleted_ls_count:0, not_start_cnt:0, interrupted_cnt:0, doing_cnt:4, stopped_cnt:0, interrupted_ls_id:{id:-1}, max_scn:{val:1755224179883806002, v:0}, checkpoint_scn:{val:1755224115293438002, v:0}, max_active_piece_id:30}) [2025-08-15 10:16:45.150128] INFO [SHARE] get_backup_dest (ob_backup_connectivity.cpp:1268) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=17] success get backup dest(sql=SELECT authorization,extension FROM __all_backup_storage_info WHERE tenant_id = 1008 AND path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog' AND endpoint = 'host=cos.ap-beijing.myqcloud.com', tenant_id=1008, backup_dest={root_path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog", storage_info:{endpoint:"host=cos.ap-beijing.myqcloud.com", access_id:"access_id=AKIDiKl3A5ZXJ959EMuL3hJ1g9PTM5lFvjhT", extension:"appid=1254236265&checksum_type=md5", type:"COS", checksum_type:1, max_iops:0, max_bandwidth:0, role_arn:0x7f1e61cb2f90, external_id:0x7f1e61cb3090}}) [2025-08-15 10:16:45.152971] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF098-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.158293] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB75-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.160838] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB75-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:45.160854] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB75-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.182970] INFO mk_parent_dir (ob_backup_io_adapter.cpp:346) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=19] found parent dir(i=130, path="cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog/piece_d1001r1p30/checkpoint") [2025-08-15 10:16:45.189664] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D4F-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.204238] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA04-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.205692] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B0-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224204918919002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224204918919003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224204918919003, v:0}}], last_dump_ts_=1755224200175088, force_dump=false) [2025-08-15 10:16:45.205718] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B0-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.208738] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B0-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.210766] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B0-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.210865] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B0-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.222472] INFO [STORAGE] print_access_storage_log (ob_storage.cpp:46) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=16] access storage op=write_single_file uri=cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog/piece_d1001r1p30/checkpoint/checkpoint_info.1755224115293438002.obarc size=0 Byte cost_ts=39470 us speed=0.00 MB/s [2025-08-15 10:16:45.222491] INFO [SHARE] write_checkpoint_file_ (ob_archive_checkpoint_mgr.cpp:255) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=14] succeed to write checkpoint file.(path={cur_pos:172, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog/piece_d1001r1p30/checkpoint/checkpoint_info.1755224115293438002.obarc"}) [2025-08-15 10:16:45.244804] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3900-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:45.247349] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3900-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:45.304508] WDIAG [STORAGE] print_access_storage_log_ (ob_i_storage.cpp:907) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=11][errcode=0] access object storage cost too much time: del_file (ob_storage.cpp:839), uri=cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog/piece_d1001r1p30/checkpoint/checkpoint_info.1755224040686907000.obarc, size=0 byte, start_time=1755224205222516, cost_ts=81991 us, speed=0.00 MB/s, is_slow=1 [2025-08-15 10:16:45.305776] INFO [SHARE] get_backup_dest (ob_backup_connectivity.cpp:1268) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=20] success get backup dest(sql=SELECT authorization,extension FROM __all_backup_storage_info WHERE tenant_id = 1008 AND path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog' AND endpoint = 'host=cos.ap-beijing.myqcloud.com', tenant_id=1008, backup_dest={root_path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog", storage_info:{endpoint:"host=cos.ap-beijing.myqcloud.com", access_id:"access_id=AKIDiKl3A5ZXJ959EMuL3hJ1g9PTM5lFvjhT", extension:"appid=1254236265&checksum_type=md5", type:"COS", checksum_type:1, max_iops:0, max_bandwidth:0, role_arn:0x7f1c74e74f90, external_id:0x7f1c74e75090}}) [2025-08-15 10:16:45.307264] INFO [SHARE] do_compare_and_swap_ (ob_inner_table_operator.cpp:937) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=19] compare and swap one column(key={key:{tenant_id:1008, dest_no:0}, incarnation:1, dest_id:1001, round_id:1, state:{status:"DOING"}, start_scn:{val:1752696006021581000, v:0}, checkpoint_scn:{val:1755224040686907000, v:0}, max_scn:{val:1755224115293438002, v:0}, compatible:{version:1}, base_piece_id:1, used_piece_id:30, piece_switch_interval:86400000000, frozen_input_bytes:13323961132, frozen_output_bytes:13323961132, active_input_bytes:117537394, active_output_bytes:117537394, deleted_input_bytes:0, deleted_output_bytes:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com", comment:""}, assignments="tenant_id = 1008, dest_no = 0, round_id = 1, incarnation = 1, dest_id = 1001, status = 'DOING', start_scn = 1752696006021581000, checkpoint_scn = 1755224115293438002, max_scn = 1755224179883806002, compatible = 1, base_piece_id = 1, used_piece_id = 30, piece_switch_interval = 86400000000, frozen_input_bytes = 13323961132, frozen_output_bytes = 13323961132, active_input_bytes = 117975128, active_output_bytes = 117975128, deleted_input_bytes = 0, deleted_output_bytes = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com', comment = ''", predicates="status = 'DOING' and round_id = 1 and checkpoint_scn = 1755224040686907000", affected_rows=1, sql=update __all_log_archive_progress set tenant_id = 1008, dest_no = 0, round_id = 1, incarnation = 1, dest_id = 1001, status = 'DOING', start_scn = 1752696006021581000, checkpoint_scn = 1755224115293438002, max_scn = 1755224179883806002, compatible = 1, base_piece_id = 1, used_piece_id = 30, piece_switch_interval = 86400000000, frozen_input_bytes = 13323961132, frozen_output_bytes = 13323961132, active_input_bytes = 117975128, active_output_bytes = 117975128, deleted_input_bytes = 0, deleted_output_bytes = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com', comment = '' where tenant_id = 1008 AND dest_no = 0 and status = 'DOING' and round_id = 1 and checkpoint_scn = 1755224040686907000) [2025-08-15 10:16:45.307425] WDIAG [STORAGE.TRANS] mvcc_sanity_check_ (ob_mvcc_row.cpp:937) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=27][errcode=-5024] find double insert node(ret=-5024, node=this=0x7f152f6344e8 trans_version={val:4611686018427387903, v:0} scn={val:4611686018427387903, v:0} tx_id={txid:141877748} prev=(nil) next=(nil) modify_count=4294967295 acc_checksum=0 version=1755222899324351 type=0 flag=128 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:6} write_epoch=2, prev=this=0x7f152f634290 trans_version={val:1755224155141757000, v:0} scn={val:1755224155141757000, v:0} tx_id={txid:141877049} prev=0x7f152f634110 next=(nil) modify_count=51 acc_checksum=1201591319 version=1755222899324351 type=0 flag=4 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:3167} write_epoch=4, snapshot_version={val:1755224205306914000, v:0}, *this={this=0x7f152f622458 latch_=locked flag=11 first_dml=DELETE last_dml=INSERT update_since_compact=4 list_head=0x7f152f634290 latest_compact_node=0x7f152f6337d0 max_trans_version={val:1755224155141757000, v:0} max_trans_id=141877049 max_elr_trans_version={val:1755224155141757000, v:0} max_elr_trans_id=141877049 latest_compact_ts=1755224054673616 last_compact_cnt=47 total_trans_node_cnt=52 max_modify_scn={val:1755224155141757000, v:0} min_modify_scn={val:1755222899325101000, v:0}}) [2025-08-15 10:16:45.307460] WDIAG [STORAGE.TRANS] mvcc_write (ob_mvcc_row.cpp:999) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=28][errcode=-5024] mvcc write failed(ret=-5024, node=this=0x7f152f6344e8 trans_version={val:4611686018427387903, v:0} scn={val:4611686018427387903, v:0} tx_id={txid:141877748} prev=(nil) next=(nil) modify_count=4294967295 acc_checksum=0 version=1755222899324351 type=0 flag=128 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:6} write_epoch=2, ctx={this:0x7f1a0fdc2dd0, ls_id:{id:1}, ls:0x7f1a567ca150, branch:0, timeout:1755224235207318, tablet_id:{id:350}, table_iter:0x7f1a0fdc1308, table_version:1746780010640136, mvcc_acc_ctx:{type:3, abs_lock_timeout_ts:1755224235207318, tx_lock_timeout_us:-1, snapshot:{version:{val:1755224205306914000, v:0}, tx_id:{txid:141877748}, scn:{branch:0, seq:4}}, tx_table_guards:{tx_table_guard:{tx_table:0x7f1a567d5b10, epoch:1, mini_cache_:[0:{}, 1:{}, 2:{}, 3:{}]}, src_tx_table_guard:{tx_table:null, epoch:-1, mini_cache_:[0:{}, 1:{}, 2:{}, 3:{}]}, src_ls_handle:{ls_map_:null, ls_:null, mod_:-1}}, tx_id:{txid:141877748}, tx_desc:{this:0x7f1e79b953e0, tx_id:{txid:141877748}, state:2, addr:"10.203.1.34:2882", tenant_id:1007, session_id:3221524038, assoc_session_id:3221524038, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:{branch:0, seq:1}, op_sn:4, alloc_ts:1755224205306914, active_ts:1755224205306914, commit_ts:-1, finish_ts:-1, timeout_us:29999994, lock_timeout_us:-1, expire_ts:1755224235306908, coord_id:{id:-1}, parts:[{id:{id:1}, addr:"10.203.1.34:2882", epoch:509200492870941, first_scn:{branch:0, seq:3}, last_scn:{branch:0, seq:3}, last_touch_ts:1755224205307119}], exec_info_reap_ts:1755224205307118, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:1727264618, cluster_version:17180067073, seq_base:1755224205307115, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, conflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, modified_tables:[], last_rc_snapshot_version:{val:1755224205306914000, v:0}, ref:2}, tx_ctx:0x7f1aa3f1a750, mem_ctx:0x7f1aa3f1bcd0, tx_scn:{branch:0, seq:6}, write_flag:{is_table_api:0, is_table_lock:0, is_mds:0, is_dml_batch_opt:0, is_insert_up:1, is_write_only_index:0, is_check_row_locked:0, is_lob_aux:0, is_skip_flush_redo:0, is_update_uk:0, is_update_pk_dop:0}, handle_start_time:-1, is_standby_read:false, lock_wait_start_ts:0, is_inited:true}, tablet_stat:{ls_id:0, tablet_id:0, query_cnt:0, merge_cnt:0, scan_logical_row_cnt:0, scan_physical_row_cnt:0, scan_micro_block_cnt:0, pushdown_micro_block_cnt:0, exist_row_total_table_cnt:0, exist_row_read_table_cnt:0, insert_row_cnt:0, update_row_cnt:0, delete_row_cnt:0}, is_read_store_ctx:false, update_full_column:false}) [2025-08-15 10:16:45.307530] WDIAG [STORAGE.TRANS] mvcc_write (ob_mvcc_engine.cpp:333) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=56][errcode=-5024] mvcc write failed(ret=-5024, arg={tx_id:{txid:141877748}, data:0x7f1a0fdbfcc0, old_row:null, modify_count:4294967295, acc_checksum:0, memstore_version:1755222899324351, seq_no:{branch:0, seq:6}, write_epoch:2, scn:{val:4611686018427387903, v:0}, column_cnt:23}) [2025-08-15 10:16:45.307549] WDIAG [STORAGE.TRANS] on_key_duplication_retry (ob_memtable_context.cpp:314) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=15][errcode=0] primary key duplication conflict(key={"BIGINT":1008},{"BIGINT":1001},{"BIGINT":1},{"BIGINT":30}, this={ObIMvccCtx={alloc_type=0 ctx_descriptor=0 min_table_version=1746780010640088 max_table_version=1746780010640136 trans_version={val:4611686018427387903, v:0} commit_version={val:0, v:0} lock_wait_start_ts=0 replay_compact_version={val:0, v:0}} end_code=0 tx_status=0 is_readonly=false ref=1 trans_id={txid:141877748} ls_id=1 row_callback[alloc:1, free:0, unsubmit:1] redo[fill:0,sync_succ:0, sync_fail:0] main_list_len=3 pending_log_size=289 callback_list:{cnt=1 stat:[tx_end=0, rollback_to=0, fast_commit=0, remove_memtable=0, ext_info_log=0] detail:[(log_epoch,length,logged,synced,appended,removed,unlog_removed,branch_removed)|0:(1,3,0,0,3,0,0,0)|]}}) [2025-08-15 10:16:45.307572] WDIAG [STORAGE] insert_row_without_rowkey_check (ob_tablet.cpp:4643) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=18][errcode=-5024] fail to set memtable(ret=-5024) [2025-08-15 10:16:45.307578] WDIAG [STORAGE] insert_row_to_tablet (ob_ls_tablet_service.cpp:5667) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=4][errcode=-5024] failed to write table row(ret=-5024, table id=350, col_descs=[cnt:23, column_id=18 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=19 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=20 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=21 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=16 {type:"TIMESTAMP", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=17 {type:"TIMESTAMP", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=22 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=23 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=24 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=25 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=26 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=27 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=28 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=29 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=30 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=31 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=32 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=33 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=34 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=35 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=36 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=37 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=38 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0], datum_row={row_flag:{flag:"INSERT", flag_type:0}, trans_id:{txid:0}, scan_index:0, mvcc_row_flag:{first:0, uncommitted:0, shadow:0, compact:0, ghost:0, last:0, reserved:0, flag:0}, snapshot_version:0, fast_filter_skipped:false, have_uncommited_row:false, group_idx:0, count:23, datum_buffer:{capacity:23, datums:0x7f1da5f13ea0, local_datums:0x7f1da5f13af0}, datums:[col_id=0:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138a4, hex: F003000000000000, int: 1008}, col_id=1:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138ac, hex: E903000000000000, int: 1001}, col_id=2:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138b4, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=3:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138bc, hex: 1E00000000000000, int: 30}, col_id=4:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138c4, hex: 222C42FB5D3C0600, int: 1755224205306914}, col_id=5:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138cc, hex: 222C42FB5D3C0600, int: 1755224205306914}, col_id=6:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138d4, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=7:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138dc, hex: 0000000000000000, int: 0}, col_id=8:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138e4, hex: 0000000000000000, int: 0}, col_id=9:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138ec, hex: C8881CAC8FBA5B18, int: 1755201606021581000}, col_id=10:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138f4, hex: 32C0438508CF5B18, int: 1755224115293438002}, col_id=11:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138fc, hex: 328D268F17CF5B18, int: 1755224179883806002}, col_id=12:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f13904, hex: C8886B3D24095C18, int: 1755288006021581000}, col_id=13:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f1390c, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=14:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f13914, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=15:{len: 4, flag: 0, null: 0, ptr: 0x7f1da5f1391c, hex: 6E6F6E65, cstr: none}, col_id=16:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f13920, hex: 5828080700000000, int: 117975128}, col_id=17:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f13928, hex: 5828080700000000, int: 117975128}, col_id=18:{len: 6, flag: 0, null: 0, ptr: 0x7f1da5f13930, hex: 414354495645, cstr: ACTIVE}, col_id=19:{len: 9, flag: 0, null: 0, ptr: 0x7f1da5f13936, hex: 415641494C41424C45, cstr: AVAILABLE}, col_id=20:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f1393f, hex: 0000000000000000, int: 0}, col_id=21:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f13947, hex: 0000000000000000, int: 0}, col_id=22:{len: 135, flag: 0, null: 0, ptr: 0x7f1da5f1394f, hex: 636F733A2F2F6261636B75702D6F6365616E626173652D313235343233363236352F6261636B75702D6F62636C75737465722F6F625F766563746F722F313732373236343631382F74656E616E745F696E6361726E6174696F6E5F312F313030382F636C6F673F686F73743D636F732E61702D6265696A696E672E6D7971636C, cstr: cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcl}]}) [2025-08-15 10:16:45.307664] WDIAG [STORAGE] insert_row (ob_ls_tablet_service.cpp:3241) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=55][errcode=-5024] failed to write row(ret=-5024) [2025-08-15 10:16:45.307694] INFO [STORAGE.TRANS] rollback (ob_memtable_context.cpp:851) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=9] memtable handle rollback to successfuly(from_seq_no={branch:0, seq:7}, to_seq_no={branch:0, seq:5}, remove_cnt=1, elapsed=0, this={ObIMvccCtx={alloc_type=0 ctx_descriptor=0 min_table_version=1746780010640088 max_table_version=1746780010640136 trans_version={val:4611686018427387903, v:0} commit_version={val:0, v:0} lock_wait_start_ts=0 replay_compact_version={val:0, v:0}} end_code=0 tx_status=0 is_readonly=false ref=0 trans_id={txid:141877748} ls_id=1 row_callback[alloc:1, free:0, unsubmit:1] redo[fill:0,sync_succ:0, sync_fail:0] main_list_len=2 pending_log_size=289 callback_list:{cnt=1 stat:[tx_end=0, rollback_to=1, fast_commit=0, remove_memtable=0, ext_info_log=0] detail:[(log_epoch,length,logged,synced,appended,removed,unlog_removed,branch_removed)|0:(1,2,0,0,3,1,1,0)|]}}) [2025-08-15 10:16:45.307891] INFO [SHARE] do_insert_or_update_row_ (ob_inner_table_operator.cpp:699) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554C0-0-0] [lt=14] insert/update one row(row={key:{tenant_id:1008, dest_id:1001, round_id:1, piece_id:30}, incarnation:1, dest_no:0, file_count:0, start_scn:{val:1755201606021581000, v:0}, checkpoint_scn:{val:1755224115293438002, v:0}, max_scn:{val:1755224179883806002, v:0}, end_scn:{val:1755288006021581000, v:0}, compatible:{version:1}, input_bytes:117975128, output_bytes:117975128, status:{status:"ACTIVE"}, file_status:0, cp_file_id:0, cp_file_offset:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com"}, affected_rows=2, sql=INSERT INTO __all_log_archive_piece_files (tenant_id, dest_id, round_id, piece_id, incarnation, dest_no, file_count, start_scn, checkpoint_scn, max_scn, end_scn, compatible, input_bytes, output_bytes, status, file_status, cp_file_id, cp_file_offset, path) VALUES (1008, 1001, 1, 30, 1, 0, 0, 1755201606021581000, 1755224115293438002, 1755224179883806002, 1755288006021581000, 1, 117975128, 117975128, 'ACTIVE', 'AVAILABLE', 0, 0, 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com') ON DUPLICATE KEY UPDATE incarnation = 1, dest_no = 0, file_count = 0, start_scn = 1755201606021581000, checkpoint_scn = 1755224115293438002, max_scn = 1755224179883806002, end_scn = 1755288006021581000, compatible = 1, input_bytes = 117975128, output_bytes = 117975128, status = 'ACTIVE', file_status = 'AVAILABLE', cp_file_id = 0, cp_file_offset = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com') [2025-08-15 10:16:45.369364] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E922-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:45.371866] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E922-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:45.372387] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5297-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:45.374901] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5297-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:45.445688] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68AE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.460431] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86CD-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:45.462281] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5336-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:45.462964] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86CD-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:45.464768] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5336-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:45.465073] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:45.465092] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:45.465101] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:45.480113] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E466-0-0] [lt=18] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}) [2025-08-15 10:16:45.480153] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E466-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:16:45.505411] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0F5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.505457] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E764-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.529512] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FDC-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:45.531990] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FDC-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:45.535771] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224203596947003, replayable_scn:1755224203596947003, readable_scn:1755224203596947003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224205533084, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224203596947003, replayable_scn:1755224203596947003, readable_scn:1755224203596947003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224205533084, ora_rowscn_=1755224204855831000, content_changed=true) [2025-08-15 10:16:45.535967] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:45.535985] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:45.536000] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:45.536535] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:45.536548] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:45.536554] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:45.542476] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:45.542497] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:45.542505] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:45.548016] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:45.548029] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:45.548034] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:45.551958] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227805550752}) [2025-08-15 10:16:45.551976] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:45.551990] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:45.553925] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:45.553937] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:45.553948] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:45.553958] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1006, cost_time_us=188, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:45.557953] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227805556759}) [2025-08-15 10:16:45.557966] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:45.557975] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:45.559170] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227805557986}) [2025-08-15 10:16:45.559184] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:45.559192] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:45.569766] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E0-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:45.572284] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E0-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:45.618585] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5C-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1011, thread_idx=0) [2025-08-15 10:16:45.619887] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1011, only_for_display=false) [2025-08-15 10:16:45.621181] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:45.635538] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:45.636451] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.636768] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224205635466, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:16:45.636791] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224210636789, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:16:45.637617] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.638766] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.638789] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1011, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:45.638807] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.638819] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1011, ls_id:{id:1}, replicas:[{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:45.638930] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=86] build ls locality stat map(tenant_id=1011, ls_id={id:1}) [2025-08-15 10:16:45.638943] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.638953] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.638960] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.638967] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:45.639047] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.639065] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.639074] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.639081] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.639087] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3801, tenant_id=1011, ls_id={id:1}) [2025-08-15 10:16:45.639096] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:45.639243] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA7-0-0] [lt=37] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.640021] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4F-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.640696] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=20809, tenant_id=1011, acc_dr_task=0) [2025-08-15 10:16:45.641869] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA7-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:45.641891] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA7-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.641943] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1012, only_for_display=false) [2025-08-15 10:16:45.642562] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4F-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:45.642580] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A4F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.643132] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:45.658296] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.659465] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.660637] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.660659] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:45.660675] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.660685] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1}, replicas:[{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:45.660797] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=86] build ls locality stat map(tenant_id=1012, ls_id={id:1}) [2025-08-15 10:16:45.660808] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.660820] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.660829] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.660837] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:45.660915] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.660928] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.660937] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.660950] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=9] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.660957] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3837, tenant_id=1012, ls_id={id:1}) [2025-08-15 10:16:45.660965] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:45.662465] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:45.684667] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.685829] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.687009] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.687033] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:45.687046] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.687057] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1001}, replicas:[{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:45.687167] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=85] build ls locality stat map(tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:16:45.687176] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.687189] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.687200] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.687212] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:45.687290] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.687306] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.687316] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.687322] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.687330] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3812, tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:16:45.687337] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:45.688830] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:45.694613] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68B1-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.705824] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDC3-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.710239] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:45.710252] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:45.710258] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:45.711063] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.712210] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.713351] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.713374] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:45.713389] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.713398] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1002}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:45.713506] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=84] build ls locality stat map(tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:16:45.713518] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.713530] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.713538] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.713549] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:45.713627] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.713640] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.713648] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.713656] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.713663] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3748, tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:16:45.713671] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:45.715208] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:45.737300] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.738465] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.739614] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.739636] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:45.739653] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.739662] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1003}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:45.739779] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=91] build ls locality stat map(tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:16:45.739790] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.739805] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.739817] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.739831] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:45.739910] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.739931] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.739944] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=10] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.739952] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.739960] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3799, tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:16:45.739969] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:45.741469] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=100755, tenant_id=1012, acc_dr_task=0) [2025-08-15 10:16:45.754523] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438023][T1006_TimerWK2_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=8] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:45.754836] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDD2-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.754967] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF09C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.794678] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841401-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.795299] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7890-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.795936] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7722-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.797547] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C57-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.797842] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841401-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.798266] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7890-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.799130] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7722-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.799840] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841401-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.799956] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841401-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.800249] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7890-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.800346] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7890-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.800390] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C57-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.801119] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7722-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.801206] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7722-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.802287] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C57-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.802374] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C57-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.804105] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC089-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.807000] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC089-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.808913] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC089-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.809105] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC089-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.833752] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888A-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.835519] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A69-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.836279] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888A-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:45.836296] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888A-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.838064] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A69-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:45.838081] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A69-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:45.852512] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D52-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.860566] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7680-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.863487] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7680-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.865666] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7680-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.865762] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7680-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:45.876850] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC65C-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.877304] INFO [RS.LB] idle (ob_root_balancer.cpp:87) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=30] root balance idle(idle_time=10000000) [2025-08-15 10:16:45.877322] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:70) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=13] start check unit migrate finish [2025-08-15 10:16:45.877335] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=6] try check migrate unit finish by tenant(tenant_id=1) [2025-08-15 10:16:45.887153] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=6] try check migrate unit finish by tenant(tenant_id=1004) [2025-08-15 10:16:45.902091] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDA-0-0] [lt=12] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1007, thread_idx=0) [2025-08-15 10:16:45.903381] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1007, only_for_display=false) [2025-08-15 10:16:45.904614] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:45.913667] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1006) [2025-08-15 10:16:45.919038] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.920206] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.921388] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.921412] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1007, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:45.921426] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.921435] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1007, ls_id:{id:1}, replicas:[{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:45.921537] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=79] build ls locality stat map(tenant_id=1007, ls_id={id:1}) [2025-08-15 10:16:45.921545] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.921556] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.921564] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.921573] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:45.921650] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.921663] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.921673] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.921683] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.921693] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3895, tenant_id=1007, ls_id={id:1}) [2025-08-15 10:16:45.921704] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:45.922783] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19402, tenant_id=1007, acc_dr_task=0) [2025-08-15 10:16:45.924025] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1008, only_for_display=false) [2025-08-15 10:16:45.925246] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:45.939080] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.939909] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1008) [2025-08-15 10:16:45.940279] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.941448] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.941470] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:45.941484] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.941496] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1}, replicas:[{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:45.941600] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=81] build ls locality stat map(tenant_id=1008, ls_id={id:1}) [2025-08-15 10:16:45.941611] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.941625] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.941636] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.941648] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:45.941727] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.941742] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.941749] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.941759] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.941767] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3862, tenant_id=1008, ls_id={id:1}) [2025-08-15 10:16:45.941778] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:45.942850] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:45.948037] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=12] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224205549878, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224203726343000, replayable_scn:1755224203726343000, readable_scn:1755224203726343000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:45.960055] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:45.961282] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224205959982, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:16:45.961305] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224210961303, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:16:45.963634] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.964789] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.965957] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.965980] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:45.965999] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.966104] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=12] try check migrate unit finish by tenant(tenant_id=1012) [2025-08-15 10:16:45.966011] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1001}, replicas:[{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:45.966138] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=98] build ls locality stat map(tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:16:45.966151] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.966165] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.966178] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=10] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.966191] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:45.966291] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=75] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.966306] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.966317] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.966327] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.966338] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=9] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3839, tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:16:45.966350] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:45.967459] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:45.989040] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.990184] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.991342] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:45.991363] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:45.991377] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.991386] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1002}, replicas:[{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:45.991493] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=83] build ls locality stat map(tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:16:45.991500] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.991512] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.991520] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:45.991527] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:45.991608] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.991620] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.991630] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.991637] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:45.991644] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3747, tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:16:45.991652] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:45.991972] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1014) [2025-08-15 10:16:45.992783] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:45.995974] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C591D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.996707] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA00-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:45.998769] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3AC-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:46.005741] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2845) [17347][TimerWK0_HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=5] need remove task(remove_task_ids=[]) [2025-08-15 10:16:46.005768] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2864) [17347][TimerWK0_HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=21] remove all timeout ddl task succeed [2025-08-15 10:16:46.014101] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:46.015360] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:46.016573] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:46.016601] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:46.016616] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:46.016630] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1003}, replicas:[{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:46.016739] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=84] build ls locality stat map(tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:16:46.016748] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:46.016762] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:46.016772] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:46.016780] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:46.016858] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:46.016873] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:46.016884] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:46.016889] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:46.016900] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=4028, tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:16:46.016907] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:46.018202] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=95406, tenant_id=1008, acc_dr_task=0) [2025-08-15 10:16:46.019710] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1016) [2025-08-15 10:16:46.047073] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=16] try check migrate unit finish by tenant(tenant_id=1018) [2025-08-15 10:16:46.073176] INFO [RS] check (ob_alter_locality_finish_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B3-0-0] [lt=15] start to check alter locality finish [2025-08-15 10:16:46.074349] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B4-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.075514] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B4-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.077839] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B6-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.079029] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B6-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.081368] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B8-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.082513] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815B8-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.084781] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815BA-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.085930] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815BA-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.088237] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815BC-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.089388] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815BC-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.091686] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815BE-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.092842] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815BE-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.095143] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C0-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.096293] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C0-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.098645] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C2-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.099798] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C2-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.100979] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C2-0-0] [lt=12] start check shrink resource pool [2025-08-15 10:16:46.100996] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C3-0-0] [lt=11] start check shrink resource pool(tenant_id=1) [2025-08-15 10:16:46.102170] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C3-0-0] [lt=4][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.103331] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C3-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.103389] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C4-0-0] [lt=14] start check shrink resource pool(tenant_id=1004) [2025-08-15 10:16:46.104510] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C4-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.104486] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E3-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.105655] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C4-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.105708] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C5-0-0] [lt=11] start check shrink resource pool(tenant_id=1006) [2025-08-15 10:16:46.106824] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C5-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.107074] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E3-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:46.107092] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E3-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.107991] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C5-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.108042] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C6-0-0] [lt=11] start check shrink resource pool(tenant_id=1008) [2025-08-15 10:16:46.108492] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5917-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.109183] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C6-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.110358] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C6-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.110409] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C7-0-0] [lt=12] start check shrink resource pool(tenant_id=1012) [2025-08-15 10:16:46.111515] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C7-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.112661] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C7-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.112715] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C8-0-0] [lt=12] start check shrink resource pool(tenant_id=1014) [2025-08-15 10:16:46.113822] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C8-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.114998] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C8-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.115057] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C9-0-0] [lt=11] start check shrink resource pool(tenant_id=1016) [2025-08-15 10:16:46.116202] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C9-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.117179] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:46.117392] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815C9-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.117445] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CA-0-0] [lt=12] start check shrink resource pool(tenant_id=1018) [2025-08-15 10:16:46.118423] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224206108536, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:16:46.118445] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224211118444, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:16:46.118464] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224211118462, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:46.118563] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CA-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.119466] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [2619609][T1011_BackupDat][T1011][YB420ACB0122-0006338CC14639D3-0-0] [lt=13] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:46.119717] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CA-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.119770] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:99) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CA-0-0] [lt=12] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.119784] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CA-0-0] [lt=10] start check primary zone [2025-08-15 10:16:46.119795] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CB-0-0] [lt=7] start check primary zone(tenant_id=1) [2025-08-15 10:16:46.120905] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CB-0-0] [lt=5][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.120962] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CC-0-0] [lt=10] start check primary zone(tenant_id=1004) [2025-08-15 10:16:46.122083] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CC-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.122134] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CD-0-0] [lt=11] start check primary zone(tenant_id=1006) [2025-08-15 10:16:46.123244] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CD-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.123293] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CE-0-0] [lt=11] start check primary zone(tenant_id=1008) [2025-08-15 10:16:46.124398] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CE-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.124449] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CF-0-0] [lt=10] start check primary zone(tenant_id=1012) [2025-08-15 10:16:46.125569] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815CF-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.125618] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D0-0-0] [lt=11] start check primary zone(tenant_id=1014) [2025-08-15 10:16:46.126722] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D0-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.126771] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D1-0-0] [lt=11] start check primary zone(tenant_id=1016) [2025-08-15 10:16:46.127888] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D1-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.127945] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=13] start check primary zone(tenant_id=1018) [2025-08-15 10:16:46.128846] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [2619608][T1011_BACKUP_SC][T1011][YB420ACB0122-0006338CC132C1CC-0-0] [lt=15] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:16:46.129070] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=6][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:46.129121] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:166) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=11] start do unit balance [2025-08-15 10:16:46.134675] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AF-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.135475] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:350) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=8] start distribute for server status change [2025-08-15 10:16:46.135576] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:401) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=13] finish distribute for server status change(ret=0) [2025-08-15 10:16:46.135587] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:200) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=9] finish do unit balance(ret=0) [2025-08-15 10:16:46.138029] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AF-0-0] [lt=87] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:46.138052] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88AF-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.140668] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0F8-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.141538] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:115) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=4] start to do tenant group unit balance [2025-08-15 10:16:46.141586] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=13] start to do tenant group unit balance(zone=zone1) [2025-08-15 10:16:46.141602] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=9] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.203.1.34:2882", disk_in_use_:3131047936, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:46.141621] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=13] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:46.141657] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=12] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:46.141675] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=9] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:46.141686] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone1) [2025-08-15 10:16:46.141697] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=8] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:46.141718] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=13] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.1.34:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:46.141762] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=34] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.203.1.34:2882", disk_in_use_:3131047936, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:46.141774] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=9] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.1.34:2882", disk_in_use_:3131047936, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:46.141792] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=12] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:46.141805] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=11] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1006, unit_config:{unit_config_id:1005, name:"config_opsone_zone1_s_2_4_ywx", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7988, unit_config:0x7f1e9aa71c38, pool:0x7f1c3beb2f18}, {unit_id:1008, tenant_id:1006, resource_pool_id:1008, unit_config:{unit_config_id:1008, name:"config_test_zone1_s_2_4_emp", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f77d8, unit_config:0x7f1e9aa71998, pool:0x7f1c3beb27a8}, {unit_id:1011, tenant_id:1008, resource_pool_id:1011, unit_config:{unit_config_id:1011, name:"config_mcp_zone1_s_4_8_tmc", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7550, unit_config:0x7f1e9aa716f8, pool:0x7f1e443fda40}, {unit_id:1016, tenant_id:1012, resource_pool_id:1016, unit_config:{unit_config_id:1017, name:"config_oms_zone1_s_4_8_gzp", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f73a0, unit_config:0x7f1e9aa71458, pool:0x7f1e443fd2d0}, {unit_id:1019, tenant_id:1014, resource_pool_id:1019, unit_config:{unit_config_id:1019, name:"config_grafana_test_zone1_s_1_2_pri", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7118, unit_config:0x7f1e9aa71298, pool:0x7f1e443fc7a8}, {unit_id:1024, tenant_id:1016, resource_pool_id:1024, unit_config:{unit_config_id:1023, name:"config_temporalio_zone1_s_2_4_ykw", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6ce0, unit_config:0x7f1e9aa70f18, pool:0x7f1e74dff2d0}, {unit_id:1026, tenant_id:1018, resource_pool_id:1026, unit_config:{unit_config_id:1027, name:"config_media_zone1_s_2_4_kmn", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6b30, unit_config:0x7f1e9aa70b98, pool:0x7f1e74dfeb60}], not_grant_units=[], available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:46.141863] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=35] start to do tenant group unit balance(zone=zone2) [2025-08-15 10:16:46.141872] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=7] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.203.4.38:2882", disk_in_use_:3114270720, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:46.141888] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:46.141909] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=12] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:46.141922] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=7] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:46.141931] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone2) [2025-08-15 10:16:46.141935] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:46.141959] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=15] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.38:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:46.141992] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=26] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.203.4.38:2882", disk_in_use_:3114270720, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:46.141999] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=5] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.38:2882", disk_in_use_:3114270720, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:46.142011] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=7] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:46.142021] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=7] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1004, unit_config:{unit_config_id:1004, name:"config_opsone_zone2_s_2_4_wag", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7b38, unit_config:0x7f1e9aa71d18, pool:0x7f1c3beb3688}, {unit_id:1009, tenant_id:1006, resource_pool_id:1009, unit_config:{unit_config_id:1009, name:"config_test_zone2_s_2_4_yey", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7700, unit_config:0x7f1e9aa718b8, pool:0x7f1c3beb23f0}, {unit_id:1010, tenant_id:1008, resource_pool_id:1010, unit_config:{unit_config_id:1010, name:"config_mcp_zone2_s_4_8_bjb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7628, unit_config:0x7f1e9aa717d8, pool:0x7f1c3beb2038}, {unit_id:1018, tenant_id:1012, resource_pool_id:1018, unit_config:{unit_config_id:1016, name:"config_oms_zone2_s_4_8_kbx", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f71f0, unit_config:0x7f1e9aa71538, pool:0x7f1e443fcb60}, {unit_id:1021, tenant_id:1014, resource_pool_id:1021, unit_config:{unit_config_id:1021, name:"config_grafana_test_zone2_s_1_2_yyc", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f6f68, unit_config:0x7f1e9aa710d8, pool:0x7f1e443fc038}, {unit_id:1023, tenant_id:1016, resource_pool_id:1023, unit_config:{unit_config_id:1022, name:"config_temporalio_zone2_s_2_4_lsv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6db8, unit_config:0x7f1e9aa70ff8, pool:0x7f1e74dff688}, {unit_id:1027, tenant_id:1018, resource_pool_id:1027, unit_config:{unit_config_id:1025, name:"config_media_zone2_s_2_4_yzd", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6a58, unit_config:0x7f1e9aa70d58, pool:0x7f1e74dfe7a8}], not_grant_units=[], available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:46.142075] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=30] start to do tenant group unit balance(zone=zone3) [2025-08-15 10:16:46.142084] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.203.4.39:2882", disk_in_use_:3013607424, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:46.142095] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=7] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:46.142112] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=8] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:46.142125] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=7] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:46.142134] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone3) [2025-08-15 10:16:46.142138] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:46.142151] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=8] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.39:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:46.142182] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=25] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.203.4.39:2882", disk_in_use_:3013607424, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:46.142188] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=4] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.39:2882", disk_in_use_:3013607424, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:46.142198] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:46.142209] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=8] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1005, unit_config:{unit_config_id:1006, name:"config_opsone_zone3_s_2_4_vxv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7a60, unit_config:0x7f1e9aa71b58, pool:0x7f1c3beb32d0}, {unit_id:1007, tenant_id:1006, resource_pool_id:1007, unit_config:{unit_config_id:1007, name:"config_test_zone3_s_2_4_mui", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f78b0, unit_config:0x7f1e9aa71a78, pool:0x7f1c3beb2b60}, {unit_id:1012, tenant_id:1008, resource_pool_id:1012, unit_config:{unit_config_id:1012, name:"config_mcp_zone3_s_4_8_seu", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7478, unit_config:0x7f1e9aa71618, pool:0x7f1e443fd688}, {unit_id:1017, tenant_id:1012, resource_pool_id:1017, unit_config:{unit_config_id:1018, name:"config_oms_zone3_s_4_8_nvb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f72c8, unit_config:0x7f1e9aa71378, pool:0x7f1e443fcf18}, {unit_id:1020, tenant_id:1014, resource_pool_id:1020, unit_config:{unit_config_id:1020, name:"config_grafana_test_zone3_s_1_2_xbz", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7040, unit_config:0x7f1e9aa711b8, pool:0x7f1e443fc3f0}, {unit_id:1022, tenant_id:1016, resource_pool_id:1022, unit_config:{unit_config_id:1024, name:"config_temporalio_zone3_s_2_4_rxz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6e90, unit_config:0x7f1e9aa70e38, pool:0x7f1e74dffa40}, {unit_id:1025, tenant_id:1018, resource_pool_id:1025, unit_config:{unit_config_id:1026, name:"config_media_zone3_s_2_4_ukz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6c08, unit_config:0x7f1e9aa70c78, pool:0x7f1e74dfef18}], not_grant_units=[], available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:46.153047] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF099-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.163619] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB76-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.166205] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB76-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:46.166223] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB76-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.189738] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D50-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.202921] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:46.204171] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224206202849, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:16:46.204207] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=28] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224211204205, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:16:46.204277] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA05-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.204986] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:46.206174] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224206204915, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:16:46.206192] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224211206191, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:46.211879] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B1-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.214878] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B1-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.217014] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B1-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.217159] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B1-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.252670] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3901-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:46.255140] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3901-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:46.268131] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [1487706][T1003_BackupDat][T1003][YB420ACB0122-0006337FE347F6D9-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:46.291937] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B63B5-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}) [2025-08-15 10:16:46.291982] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B63B5-0-0] [lt=29] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:16:46.299124] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [1487705][T1003_BACKUP_SC][T1003][YB420ACB0122-0006337FE332C1CC-0-0] [lt=13] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:16:46.377258] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E923-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:46.379752] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E923-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:46.380168] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5298-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:46.382699] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5298-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:46.445764] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68AF-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.463878] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:46.463889] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:46.463897] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:46.468329] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86CE-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:46.470052] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5337-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:46.470828] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86CE-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:46.472521] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5337-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:46.505485] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0F6-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.505524] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E765-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.515905] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF236-0-0] [lt=10] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.535970] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:46.535986] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:46.535991] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:46.536614] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:46.536625] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:46.536633] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:46.537314] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FDD-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:46.539805] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FDD-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:46.541934] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:46.541958] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:46.541968] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:46.547910] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5852-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81054688GB}) [2025-08-15 10:16:46.547954] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5852-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:16:46.548239] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:46.548251] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:46.548257] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:46.553483] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:46.553499] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:46.553509] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:46.577721] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E1-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:46.580241] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E1-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:46.625227] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [728059][T1017_BackupDat][T1017][YB420ACB0122-000633A355859847-0-0] [lt=16] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:16:46.644638] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA8-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.645231] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A50-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.647225] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA8-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:46.647244] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA8-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.647730] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A50-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:46.647747] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A50-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.694673] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68B2-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.705898] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDC4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.710321] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:46.710334] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:46.710340] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:46.754904] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDD3-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.755040] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF09D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.794327] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:46.800994] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841402-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224206420925004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224206420925004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224206420925004, v:0}}], last_dump_ts_=1755224201769896, force_dump=false) [2025-08-15 10:16:46.801019] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841402-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.801536] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7891-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.802179] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7723-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.803332] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C58-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.804186] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841402-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224206420925004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224206420925004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224206420925004, v:0}}], last_dump_ts_=1755224201773003, force_dump=false) [2025-08-15 10:16:46.804211] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841402-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.804507] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7891-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.805380] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7723-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.806266] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C58-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.806341] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841402-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.806424] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7891-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.806451] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841402-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.806499] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7891-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.807415] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7723-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.807517] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7723-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.808178] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C58-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.808251] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C58-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.809991] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.812896] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08A-0-0] [lt=15] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224206677601000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224206677601001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224206677601001, v:0}}], last_dump_ts_=1755224201783725, force_dump=false) [2025-08-15 10:16:46.812918] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08A-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.814821] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08A-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.814901] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.838998] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888B-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.840730] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6A-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.841544] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888B-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:46.841561] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.843248] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6A-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:46.843265] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:46.851389] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:46.852588] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D53-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.852604] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224206845092, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:16:46.852627] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224211852626, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:16:46.866612] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:16:46.866734] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7681-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.869665] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7681-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.871552] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7681-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.871643] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7681-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:46.996052] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C591E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:46.996779] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA01-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.000149] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3AD-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:47.108569] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5918-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.109828] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E4-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.112374] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E4-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:47.112391] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E4-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.140745] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0F9-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.140831] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B0-0-0] [lt=42] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.143413] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B0-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:47.143432] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B0-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.153124] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF09A-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.169004] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB77-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.171551] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB77-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:47.171567] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB77-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.189810] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D51-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.200506] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [727133][T1016_TimerWK1_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=12] mview_mds_op(tx_count=0) [2025-08-15 10:16:47.200517] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [727133][T1016_TimerWK1_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=8] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=837) [2025-08-15 10:16:47.201729] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727135][T1016_TimerWK3_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=44] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:47.204352] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA06-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.218145] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B2-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.220033] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:47.221282] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B2-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.223247] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B2-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.223341] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B2-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.260431] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3902-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:47.262889] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3902-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:47.274328] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619488][T1012_TimerWK3_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=34] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:47.334607] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487604][T1004_TimerWK1_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:47.349226] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:47.350459] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224207349156, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:16:47.350482] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224212350480, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:16:47.385113] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E924-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:47.387606] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E924-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:47.387899] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5299-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:47.390337] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5299-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:47.440749] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:47.442003] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224207440676, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:16:47.442026] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224212442025, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:16:47.445839] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68B0-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.457664] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E66-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224207456984000, start_id=1755224207456984000, end_id=1755224207456984001) [2025-08-15 10:16:47.458314] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E66-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224207456984000, v:0}, tenant_id=1017) [2025-08-15 10:16:47.463791] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224205920172003, replayable_scn:1755224205920172003, readable_scn:1755224205920172003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224207461198, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224205920172003, replayable_scn:1755224205920172003, readable_scn:1755224205920172003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224207461198, ora_rowscn_=1755224207220281000, content_changed=true) [2025-08-15 10:16:47.463974] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:47.463995] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:47.464002] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:47.476184] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86CF-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:47.477784] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5338-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:47.479848] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86CF-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:47.480288] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5338-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:47.480653] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A0A7F-0-0] [lt=12] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}) [2025-08-15 10:16:47.480691] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A0A7F-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:16:47.505561] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0F7-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.505599] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E766-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.536009] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:47.536022] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:47.536028] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:47.536497] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224205419418005, replayable_scn:1755224205419418005, readable_scn:1755224205419418005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224207533734, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224205419418005, replayable_scn:1755224205419418005, readable_scn:1755224205419418005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224207533734, ora_rowscn_=1755224206802789000, content_changed=true) [2025-08-15 10:16:47.536677] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:47.536687] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:47.536695] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:47.538007] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:47.541840] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224205419418003, replayable_scn:1755224205419418003, readable_scn:1755224205419418003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224207539002, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224205419418003, replayable_scn:1755224205419418003, readable_scn:1755224205419418003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224207539002, ora_rowscn_=1755224206803843000, content_changed=true) [2025-08-15 10:16:47.542005] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:47.542017] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:47.542022] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:47.543272] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:47.545147] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FDE-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:47.547614] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FDE-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:47.547984] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224205419418004, replayable_scn:1755224205419418004, readable_scn:1755224205419418004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224207545324, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224205419418004, replayable_scn:1755224205419418004, readable_scn:1755224205419418004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224207545324, ora_rowscn_=1755224206804896000, content_changed=true) [2025-08-15 10:16:47.548161] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:47.548172] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:47.548178] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:47.549476] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:47.553373] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224205753991000, replayable_scn:1755224205753991000, readable_scn:1755224205753991000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224207550592, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224205753991000, replayable_scn:1755224205753991000, readable_scn:1755224205753991000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224207550592, ora_rowscn_=1755224206811211000, content_changed=true) [2025-08-15 10:16:47.553535] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:47.553546] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:47.553553] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:47.585574] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E2-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:47.588042] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E2-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:47.592428] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E6F-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224207591653000, start_id=1755224207591653000, end_id=1755224207591653001) [2025-08-15 10:16:47.593109] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E6F-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224207591653000, v:0}, tenant_id=1015) [2025-08-15 10:16:47.593191] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA671-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224207592708000, start_id=1755224207592708000, end_id=1755224207592708001) [2025-08-15 10:16:47.593877] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA671-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224207592708000, v:0}, tenant_id=1005) [2025-08-15 10:16:47.650031] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA9-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.650414] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A51-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.652588] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA9-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:47.652606] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFA9-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.652914] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A51-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:47.652931] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A51-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.694746] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68B3-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.705947] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDC5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.710217] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=12] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224205419418003, replayable_scn:1755224205419418003, readable_scn:1755224205419418003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224207707293, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224205419418003, replayable_scn:1755224205419418003, readable_scn:1755224205419418003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224207707293, ora_rowscn_=1755224206802789000, content_changed=true) [2025-08-15 10:16:47.710397] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:47.710410] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:47.710419] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:47.754980] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDD4-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.755109] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF09E-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.807377] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841403-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.807496] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7892-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.808392] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7724-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.809183] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C59-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.810419] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841403-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.810545] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7892-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224207422295003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224207422295003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224207422295003, v:0}}], last_dump_ts_=1755224202780111, force_dump=false) [2025-08-15 10:16:47.810567] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7892-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.811488] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7724-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.812289] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C59-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.812394] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841403-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.812475] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7892-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.812502] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841403-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.812573] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7892-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.813537] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7724-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.813637] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7724-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.814180] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:47.814237] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C59-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.814331] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C59-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.815397] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224207814106, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:16:47.815418] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224212815417, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:16:47.815456] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:47.815885] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08B-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224207422295003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224207422295004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224207422295004, v:0}}], last_dump_ts_=1755224202786652, force_dump=false) [2025-08-15 10:16:47.815916] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08B-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.816604] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224207815390, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:16:47.816623] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224212816622, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:16:47.818746] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08B-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.820569] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.820651] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.821564] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36CF-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224207820908000, start_id=1755224207820908000, end_id=1755224207820908001) [2025-08-15 10:16:47.822169] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36CF-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224207820908000, v:0}, tenant_id=1011) [2025-08-15 10:16:47.844306] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888C-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.845919] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6B-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.846821] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888C-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:47.846837] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888C-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.848481] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6B-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:47.848496] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6B-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:47.852660] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D54-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.872594] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7682-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.875687] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7682-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.877621] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7682-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.877718] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7682-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:47.881804] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC65D-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.940132] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB7-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224207939766000, start_id=1755224207939766000, end_id=1755224207939766001) [2025-08-15 10:16:47.940787] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB7-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224207939766000, v:0}, tenant_id=1013) [2025-08-15 10:16:47.977510] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:47.978740] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224207977437, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:16:47.978761] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224212978760, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:16:47.996124] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C591F-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:47.996856] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA02-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.001538] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3AE-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:48.063598] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:48.064841] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224208063521, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:16:48.064864] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224213064863, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:16:48.108646] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5919-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.114548] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E5-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.117117] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E5-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:48.117135] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E5-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.121090] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B7-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224208120688000, start_id=1755224208120688000, end_id=1755224208120688001) [2025-08-15 10:16:48.121925] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B7-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224208120688000, v:0}, tenant_id=1012) [2025-08-15 10:16:48.140151] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463BF-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224208139606000, start_id=1755224208139606000, end_id=1755224208139606001) [2025-08-15 10:16:48.140819] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0FA-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.140933] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463BF-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224208139606000, v:0}, tenant_id=1) [2025-08-15 10:16:48.146164] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B1-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.148681] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B1-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:48.148698] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B1-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.153199] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF09B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.174292] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB78-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.176816] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB78-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:48.176832] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB78-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.189882] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D52-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.204428] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA07-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.224331] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B3-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.227592] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B3-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.229670] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B3-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.229794] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B3-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.268499] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3903-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:48.271010] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3903-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:48.292396] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D8C-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}) [2025-08-15 10:16:48.292433] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D8C-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:16:48.324525] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A631-0-0] [lt=9] start do ObArchiveSchedulerService round(round=124006) [2025-08-15 10:16:48.328143] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A631-0-0] [lt=12][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:16:48.328158] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A631-0-0] [lt=11][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1012 AND dest_no = 0 AND name = 'state', exec_tenant_id=1011) [2025-08-15 10:16:48.328201] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A631-0-0] [lt=8][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1012, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:16:48.328215] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A631-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1012, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:16:48.328225] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A631-0-0] [lt=8][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1012, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:16:48.392891] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E925-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:48.395608] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD529A-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:48.396526] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E925-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:48.398058] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD529A-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:48.445911] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68B1-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.463935] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:48.463955] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:48.463960] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:48.485181] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D0-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:48.485634] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5339-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:48.487658] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D0-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:48.488080] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5339-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:48.505636] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0F8-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.505673] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E767-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.516195] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF237-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.535970] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224206601877003, replayable_scn:1755224206601877003, readable_scn:1755224206601877003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224208533062, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224206601877003, replayable_scn:1755224206601877003, readable_scn:1755224206601877003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224208533062, ora_rowscn_=1755224207874547000, content_changed=true) [2025-08-15 10:16:48.536135] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:48.536148] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:48.536153] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:48.536711] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:48.536722] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:48.536730] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:48.542074] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:48.542090] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:48.542096] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:48.548230] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:48.548248] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:48.548253] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:48.548360] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5854-0-0] [lt=6] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81054688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81445312GB}) [2025-08-15 10:16:48.548399] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5854-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:16:48.552926] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FDF-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:48.553608] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:48.553619] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:48.553626] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:48.553636] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1006, cost_time_us=191, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:48.555407] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FDF-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:48.563499] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227808562317}) [2025-08-15 10:16:48.563514] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:48.563525] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:48.569428] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227808568257}) [2025-08-15 10:16:48.569443] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:48.569454] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:48.570647] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227808569466}) [2025-08-15 10:16:48.570660] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:48.570670] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:48.593366] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E3-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:48.595862] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E3-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:48.621246] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:48.622472] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224208621170, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:16:48.622494] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224213622492, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:16:48.655348] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAA-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.655574] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A52-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.657897] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAA-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:48.657917] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAA-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.658118] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A52-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:48.658137] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A52-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.694821] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68B4-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.706026] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDC6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.710459] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:48.710470] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:48.710475] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:48.733730] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF5-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1, thread_idx=0) [2025-08-15 10:16:48.734985] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-08-15 10:16:48.736208] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:48.750021] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:48.751161] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:48.752292] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:48.752313] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:48.752328] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.752337] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:48.752437] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=79] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-08-15 10:16:48.752445] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:48.752456] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:48.752464] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:48.752473] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:48.752549] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.752564] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.752573] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.752580] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.752587] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3731, tenant_id=1, ls_id={id:1}) [2025-08-15 10:16:48.752595] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:48.753667] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18682, tenant_id=1, acc_dr_task=0) [2025-08-15 10:16:48.755054] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDD5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.755179] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF09F-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.775745] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5F-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224208774838000, start_id=1755224208774838000, end_id=1755224208774838001) [2025-08-15 10:16:48.776390] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5F-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224208774838000, v:0}, tenant_id=1016) [2025-08-15 10:16:48.783141] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [438238][T1006_MergeSche][T1006][YB420ACB0122-000633820812DE6F-0-0] [lt=65] zone_merge_mgr is already loaded(tenant_id=1006, global_merge_info={tenant_id:1006, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402201372, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194784919589, need_update:false}}, zone_merge_infos=[{tenant_id:1006, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194784921725, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402224483, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1006, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194784925063, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402226936, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1006, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194784926922, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402229083, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:16:48.813613] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841404-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.813680] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7893-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.814636] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7725-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224208424721002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224208424721003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224208424721003, v:0}}], last_dump_ts_=1755224203782489, force_dump=false) [2025-08-15 10:16:48.814661] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7725-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.815264] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.816682] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841404-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.816691] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7893-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.817828] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7725-0-0] [lt=15] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224208424721002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224208424721003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224208424721003, v:0}}], last_dump_ts_=1755224203785529, force_dump=false) [2025-08-15 10:16:48.817856] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7725-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.818250] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.818645] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7893-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.818710] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841404-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.818723] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7893-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.818808] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841404-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.819972] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7725-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.820118] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7725-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.820170] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.820245] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.821567] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08C-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.824461] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08C-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.826340] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.826427] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08C-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.849553] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888D-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.851143] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6C-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.852098] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888D-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:48.852115] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.852733] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D55-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.853646] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6C-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:48.853663] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6C-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:48.878667] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7683-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.881645] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7683-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.883861] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7683-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.883964] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7683-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:48.926324] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8276-0-0] [lt=13] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1005, thread_idx=0) [2025-08-15 10:16:48.927598] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1005, only_for_display=false) [2025-08-15 10:16:48.928827] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:48.942735] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:48.943898] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:48.945063] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:48.945090] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1005, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:48.945109] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.945120] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1005, ls_id:{id:1}, replicas:[{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:48.945223] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=80] build ls locality stat map(tenant_id=1005, ls_id={id:1}) [2025-08-15 10:16:48.945231] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:48.945245] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:48.945253] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:48.945264] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:48.945361] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=73] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.945376] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.945386] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.945390] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.945399] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3814, tenant_id=1005, ls_id={id:1}) [2025-08-15 10:16:48.945410] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:48.946468] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18871, tenant_id=1005, acc_dr_task=0) [2025-08-15 10:16:48.947696] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1006, only_for_display=false) [2025-08-15 10:16:48.948887] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:48.949560] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224208549912, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224206677601001, replayable_scn:1755224206677601001, readable_scn:1755224206677601001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:48.962630] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:48.963790] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:48.964953] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:48.964976] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:48.964994] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.965006] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1}, replicas:[{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:48.965108] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=80] build ls locality stat map(tenant_id=1006, ls_id={id:1}) [2025-08-15 10:16:48.965116] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:48.965130] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:48.965136] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:48.965147] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:48.965235] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=65] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.965249] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.965259] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.965264] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.965273] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3782, tenant_id=1006, ls_id={id:1}) [2025-08-15 10:16:48.965283] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:48.966361] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:48.987220] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:48.988376] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:48.989530] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:48.989553] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:48.989570] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.989582] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1001}, replicas:[{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:48.989690] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=85] build ls locality stat map(tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:16:48.989698] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:48.989711] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:48.989721] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:48.989729] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:48.989806] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.989821] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.989831] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.989837] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:48.989847] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3767, tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:16:48.989858] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:48.990969] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:48.996196] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5920-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:48.996928] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA03-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.000240] INFO [RS] process (ob_root_inspection.cpp:410) [17454][RSInspectTask][T0][Y0-000634393DD76AF7-0-0] [lt=12] purge recyclebin task begin [2025-08-15 10:16:49.000258] INFO [RS] process (ob_root_inspection.cpp:428) [17454][RSInspectTask][T0][Y0-000634393DD76AF7-0-0] [lt=11] submit purge recyclebin task success(delay=60000000) [2025-08-15 10:16:49.000268] INFO [RS] process (ob_root_inspection.cpp:430) [17454][RSInspectTask][T0][Y0-000634393DD76AF7-0-0] [lt=8] purge recyclebin task end(delay=60000000) [2025-08-15 10:16:49.002933] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3AF-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:49.011920] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.013078] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.014242] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.014264] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:49.014280] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.014289] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1002}, replicas:[{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:49.014390] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=78] build ls locality stat map(tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:16:49.014398] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.014409] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.014418] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.014427] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:49.014501] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.014514] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.014521] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.014529] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.014536] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3761, tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:16:49.014544] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:49.015658] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:49.036669] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.037826] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.038993] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.039014] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:49.039029] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.039039] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1003}, replicas:[{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:49.039140] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=79] build ls locality stat map(tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:16:49.039150] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.039161] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.039170] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.039178] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:49.039253] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.039264] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.039272] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.039279] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.039285] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3766, tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:16:49.039293] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:49.040364] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93882, tenant_id=1006, acc_dr_task=0) [2025-08-15 10:16:49.108721] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C591A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.119299] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E6-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.121843] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E6-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:49.121865] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E6-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.140893] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0FB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.151471] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B2-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.153228] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF09C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.154050] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B2-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:49.154067] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B2-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.179675] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB79-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.182276] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB79-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:49.182294] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB79-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.189953] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D53-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.192451] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA652-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224209191471000, start_id=1755224209191471000, end_id=1755224209191471001) [2025-08-15 10:16:49.193278] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA652-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224209191471000, v:0}, tenant_id=1006) [2025-08-15 10:16:49.195777] INFO [SHARE] update_decay_factor_ (ob_throttle_unit.ipp:284) [438237][T1006_FrzInfoDe][T1005][YB420ACB0122-0006338207BFA652-0-0] [lt=18] [Throttle] Update Throttle Unit Config(is_adaptive_update=true, N=5.208583498001098633e+01, this=0x7f1abcf83208, enable_adaptive_limit_=true, Unit Name=TxShare, Config Specify Resource Limit(MB)=511, Resource Limit(MB)=260, Throttle Trigger(MB)=156, Throttle Percentage=60, Max Duration(us)=7200000000, Decay Factor=3.767002582009702564e+03) [2025-08-15 10:16:49.204501] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA08-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.230714] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B4-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.233762] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B4-0-0] [lt=15] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224208925282002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224208925282002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224208925282002, v:0}}], last_dump_ts_=1755224204202692, force_dump=false) [2025-08-15 10:16:49.233782] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B4-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.235862] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B4-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.235966] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B4-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.257361] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2AB-0-0] [lt=61] get number(ret=0, service_type_=0, range=1, base_id=1755224209256700000, start_id=1755224209256700000, end_id=1755224209256700001) [2025-08-15 10:16:49.258188] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2AB-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224209256700000, v:0}, tenant_id=1008) [2025-08-15 10:16:49.276333] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3904-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:49.278808] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3904-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:49.401777] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E926-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:49.403241] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD529B-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:49.404287] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E926-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:49.405733] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD529B-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:49.419849] INFO [RS] prepare (ob_mview_maintenance_task.cpp:176) [1487606][T1004_TimerWK3_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=9] mview maintenance task prepare success(tenant_id_=1004, round_=116351, prepare_cost_us_=0) [2025-08-15 10:16:49.445986] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68B2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.463985] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:49.463999] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:49.464005] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:49.465270] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:49.481199] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A0A81-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}) [2025-08-15 10:16:49.481239] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A0A81-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:16:49.492896] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D1-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:49.493286] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD533A-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:49.495395] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D1-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:49.495808] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD533A-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:49.505711] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0F9-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.505754] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E768-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.536130] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:49.536143] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:49.536148] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:49.536804] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:49.536816] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:49.536824] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:49.537423] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:49.542130] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:49.542143] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:49.542148] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:49.548220] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:49.548235] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:49.548242] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:49.553625] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:49.553637] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:49.553642] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:49.554913] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:49.560621] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE0-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:49.563090] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE0-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:49.601119] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E4-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:49.603554] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E4-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:49.641718] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:49.642955] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224209641640, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:16:49.642977] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224214642975, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:16:49.660656] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAB-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.660761] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A53-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.663227] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAB-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:49.663246] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A53-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:49.663260] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A53-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.663251] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAB-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.694899] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68B5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.706102] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDC7-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.710468] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:49.710482] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:49.710488] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:49.711811] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:49.750109] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5EB-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224209750065000, start_id=1755224209750065000, end_id=1755224209750065001) [2025-08-15 10:16:49.750830] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5EB-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224209750065000, v:0}, tenant_id=1003) [2025-08-15 10:16:49.755121] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDD6-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.755248] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF0A0-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.785578] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C399E-0-0] [lt=11] [RESTORE] restore scheduler quit [2025-08-15 10:16:49.786866] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C399E-0-0] [lt=12] get targets(ret=0, jobs=[], exec_tenant_id=1003, sql=select * from __all_recover_table_job) [2025-08-15 10:16:49.788063] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C399F-0-0] [lt=13] get targets(ret=0, jobs=[], exec_tenant_id=1003, sql=select * from __all_import_table_job) [2025-08-15 10:16:49.788085] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C39A0-0-0] [lt=13][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224209788080, ref_timestamp=1755224149783951, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:16:49.788166] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C39A1-0-0] [lt=10] [RESTORE] restore scheduler start [2025-08-15 10:16:49.788173] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C39A2-0-0] [lt=6] [RESTORE] try process restore job [2025-08-15 10:16:49.789314] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C39A2-0-0] [lt=5] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:16:49.819804] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841405-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.819804] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7894-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224209426107002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224209766880000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224209766880000, v:0}}], last_dump_ts_=1755224204789179, force_dump=false) [2025-08-15 10:16:49.819830] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7894-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.821021] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7726-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.821172] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5B-0-0] [lt=5] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224209426107002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224209426107003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224209426107003, v:0}}], last_dump_ts_=1755224204790935, force_dump=false) [2025-08-15 10:16:49.821202] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5B-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.822868] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841405-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.823004] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7894-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.823986] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7726-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.824240] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5B-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224209426107002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224209426107002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224209426107003, v:0}}], last_dump_ts_=1755224204794045, force_dump=false) [2025-08-15 10:16:49.824264] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5B-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.824919] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841405-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.825040] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841405-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.825044] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7894-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.825151] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7894-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.825900] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7726-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.825979] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7726-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.826265] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5B-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.826354] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5B-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.827323] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.830148] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08D-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.832107] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.832216] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.852806] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D56-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.854786] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.856283] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6D-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.857317] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888E-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:49.857335] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888E-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.858783] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6D-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:49.858802] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:49.884900] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7684-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224209752173000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224209752173001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224209752173001, v:0}}], last_dump_ts_=1755224204854511, force_dump=false) [2025-08-15 10:16:49.884923] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7684-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.887823] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC65E-0-0] [lt=13] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.887843] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7684-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224209752173000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224209752173001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224209752173001, v:0}}], last_dump_ts_=1755224204857554, force_dump=false) [2025-08-15 10:16:49.887865] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7684-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.889747] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7684-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.889838] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7684-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:49.892775] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C8-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224209891901000, start_id=1755224209891901000, end_id=1755224209891901001) [2025-08-15 10:16:49.893573] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C8-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224209891901000, v:0}, tenant_id=1004) [2025-08-15 10:16:49.897408] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E68-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1003, thread_idx=0) [2025-08-15 10:16:49.898690] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1003, only_for_display=false) [2025-08-15 10:16:49.899902] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:49.913732] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.914885] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.916030] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.916052] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1003, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:49.916066] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.916075] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1003, ls_id:{id:1}, replicas:[{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:49.916182] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=83] build ls locality stat map(tenant_id=1003, ls_id={id:1}) [2025-08-15 10:16:49.916190] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.916201] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.916209] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.916218] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:49.916293] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.916305] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.916313] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.916321] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.916325] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3732, tenant_id=1003, ls_id={id:1}) [2025-08-15 10:16:49.916333] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:49.917410] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18721, tenant_id=1003, acc_dr_task=0) [2025-08-15 10:16:49.918615] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1004, only_for_display=false) [2025-08-15 10:16:49.919818] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:49.933517] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.934665] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.935808] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.935832] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:49.935847] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.935856] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:49.935964] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=84] build ls locality stat map(tenant_id=1004, ls_id={id:1}) [2025-08-15 10:16:49.935972] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.935983] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.935991] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.936000] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:49.936075] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.936084] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.936091] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.936099] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.936103] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3732, tenant_id=1004, ls_id={id:1}) [2025-08-15 10:16:49.936111] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:49.937166] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:49.957727] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.958876] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.960025] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.960047] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:49.960063] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.960075] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:49.960183] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=85] build ls locality stat map(tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:16:49.960191] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.960203] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.960214] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.960226] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:49.960302] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.960317] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.960326] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.960335] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.960340] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3763, tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:16:49.960351] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:49.961406] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:49.981970] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.983154] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.984357] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:49.984378] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:49.984393] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.984402] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1002}, replicas:[{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:49.984503] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=80] build ls locality stat map(tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:16:49.984511] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.984521] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.984529] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:49.984538] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:49.984619] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.984631] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.984639] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.984646] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:49.984653] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3835, tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:16:49.984661] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:49.985714] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:49.996271] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5921-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:49.997008] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA04-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.004341] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3B0-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:50.006318] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.007459] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.008606] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.008628] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:50.008646] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.008657] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1003}, replicas:[{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:50.008761] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=81] build ls locality stat map(tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:16:50.008769] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.008782] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.008793] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.008804] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:50.008881] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.008898] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.008909] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.008919] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.008924] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3806, tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:16:50.008934] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:50.010014] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E69-0-0] [lt=8] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=92590, tenant_id=1004, acc_dr_task=0) [2025-08-15 10:16:50.018391] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17739][T1_TimerWK2_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:50.108795] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C591B-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.124587] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E7-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.127146] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E7-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:50.127167] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E7-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.140970] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0FC-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.153301] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF09D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.156753] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B3-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.159305] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B3-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:50.159322] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B3-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.185020] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7A-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.187517] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7A-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:50.187532] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7A-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.190025] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D54-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.204573] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA09-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.222064] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C9-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224210221284000, start_id=1755224210221284000, end_id=1755224210221284001) [2025-08-15 10:16:50.222793] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C9-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224210221284000, v:0}, tenant_id=1007) [2025-08-15 10:16:50.222813] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C9-0-0] [lt=12] zone_merge_mgr is already loaded(tenant_id=1007, global_merge_info={tenant_id:1007, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194404313977, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194707257901, need_update:false}}, zone_merge_infos=[{tenant_id:1007, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194707260697, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194404343035, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1007, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194707264633, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194404359873, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1007, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194707266479, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194404364879, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:16:50.233110] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C1-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1017, thread_idx=0) [2025-08-15 10:16:50.234390] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1017, only_for_display=false) [2025-08-15 10:16:50.235617] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:50.236979] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B5-0-0] [lt=11] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224209926741002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224209926741002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224209926741002, v:0}}], last_dump_ts_=1755224205205691, force_dump=false) [2025-08-15 10:16:50.237012] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B5-0-0] [lt=25] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.240413] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B5-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.242434] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B5-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.242643] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B5-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.249450] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.250589] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.251730] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.251753] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1017, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:50.251771] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.251784] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1017, ls_id:{id:1}, replicas:[{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:50.251891] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=83] build ls locality stat map(tenant_id=1017, ls_id={id:1}) [2025-08-15 10:16:50.251899] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.251912] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.251923] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.251934] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:50.252017] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.252032] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.252044] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=10] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.252049] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.252057] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3799, tenant_id=1017, ls_id={id:1}) [2025-08-15 10:16:50.252068] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:50.253132] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18742, tenant_id=1017, acc_dr_task=0) [2025-08-15 10:16:50.254322] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1018, only_for_display=false) [2025-08-15 10:16:50.255504] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:50.269100] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.270247] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.271389] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.271413] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:50.271429] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.271439] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1}, replicas:[{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:50.271542] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1}) [2025-08-15 10:16:50.271549] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.271563] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.271573] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.271585] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:50.271662] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.271676] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.271686] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.271692] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.271700] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3738, tenant_id=1018, ls_id={id:1}) [2025-08-15 10:16:50.271710] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:50.272806] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:50.284090] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3905-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:50.286625] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3905-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:50.292888] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D8D-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}) [2025-08-15 10:16:50.292930] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D8D-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:16:50.293454] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.294602] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.295732] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.295754] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:50.295768] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.295779] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1001}, replicas:[{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:50.295880] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=78] build ls locality stat map(tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:16:50.295888] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.295901] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.295913] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.295924] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:50.296006] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.296021] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.296030] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.296036] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.296044] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3732, tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:16:50.296054] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:50.297113] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:50.317767] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.318906] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.320040] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.320063] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:50.320080] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.320091] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1002}, replicas:[{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:50.320194] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=81] build ls locality stat map(tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:16:50.320202] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.320214] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.320225] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.320236] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:50.320313] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.320326] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.320335] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.320340] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.320349] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3730, tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:16:50.320355] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:50.321442] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:50.342026] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.343176] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.344304] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:50.344327] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:50.344344] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.344355] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1003}, replicas:[{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:50.344466] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=87] build ls locality stat map(tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:16:50.344474] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.344486] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.344496] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:50.344506] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:50.344581] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.344595] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.344604] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.344608] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:50.344617] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3734, tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:16:50.344625] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:50.345686] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68C2-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=92540, tenant_id=1018, acc_dr_task=0) [2025-08-15 10:16:50.409634] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E927-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:50.411068] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD529C-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:50.412106] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E927-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:50.413557] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD529C-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:50.446060] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68B3-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.463954] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=14] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224208925282002, replayable_scn:1755224208925282002, readable_scn:1755224208925282002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224210460898, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224208925282002, replayable_scn:1755224208925282002, readable_scn:1755224208925282002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224210460898, ora_rowscn_=1755224210239190000, content_changed=true) [2025-08-15 10:16:50.464138] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:50.464150] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:50.464156] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:50.500702] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D2-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:50.502357] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD533B-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:50.503184] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D2-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:50.504818] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD533B-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:50.505787] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0FA-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.505832] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E769-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.516476] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF238-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.536163] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:50.536177] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:50.536182] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:50.536681] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224208424721003, replayable_scn:1755224208424721003, readable_scn:1755224208424721003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224210533387, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224208424721003, replayable_scn:1755224208424721003, readable_scn:1755224208424721003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224210533387, ora_rowscn_=1755224209821472000, content_changed=true) [2025-08-15 10:16:50.536848] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:50.536858] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:50.536865] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:50.542017] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224208424721003, replayable_scn:1755224208424721003, readable_scn:1755224208424721003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224210539705, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224208424721003, replayable_scn:1755224208424721003, readable_scn:1755224208424721003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224210539705, ora_rowscn_=1755224209822489000, content_changed=true) [2025-08-15 10:16:50.542176] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:50.542189] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:50.542194] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:50.548121] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224208423668004, replayable_scn:1755224208423668004, readable_scn:1755224208423668004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224210544935, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224208423668004, replayable_scn:1755224208423668004, readable_scn:1755224208423668004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224210544935, ora_rowscn_=1755224209822489000, content_changed=true) [2025-08-15 10:16:50.548295] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:50.548305] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:50.548312] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:50.548845] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAD4-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81445312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81835938GB}) [2025-08-15 10:16:50.548887] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAD4-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:16:50.553559] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224208728576000, replayable_scn:1755224208728576000, readable_scn:1755224208728576000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224210550180, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224208728576000, replayable_scn:1755224208728576000, readable_scn:1755224208728576000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224210550180, ora_rowscn_=1755224209828825000, content_changed=true) [2025-08-15 10:16:50.553747] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:50.553768] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:50.553780] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:50.568386] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE1-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:50.570900] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE1-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:50.608881] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E5-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:50.611716] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E5-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:50.636872] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:50.638097] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224210636789, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:16:50.638122] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224215638119, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:16:50.665971] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A54-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.665976] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAC-0-0] [lt=46] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.668513] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A54-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:50.668529] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A54-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.668561] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAC-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:50.668574] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAC-0-0] [lt=10] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.694977] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68B6-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.706166] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDC8-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.710288] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224208423668003, replayable_scn:1755224208423668003, readable_scn:1755224208423668003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224210706771, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224208423668003, replayable_scn:1755224208423668003, readable_scn:1755224208423668003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224210706771, ora_rowscn_=1755224209821472000, content_changed=true) [2025-08-15 10:16:50.710445] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:50.710460] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:50.710466] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:50.755191] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDD7-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.755319] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF0A1-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.759773] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [3414398][T1006_TimerWK4_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=19] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:50.825981] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841406-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.826106] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7895-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.826886] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7727-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.827313] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.829039] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7895-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.829117] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841406-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.830040] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7727-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.830204] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.831025] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7895-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.831051] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841406-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.831125] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7895-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.831148] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841406-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.832120] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7727-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.832216] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7727-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.832289] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.832465] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.833140] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.836135] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08E-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.838054] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.838160] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.852880] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D57-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.860020] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888F-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.861439] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6E-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.862528] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888F-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:50.862548] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D888F-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.863920] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6E-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:50.863937] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:50.890771] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7685-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.893730] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7685-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.895732] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7685-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.895833] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7685-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:50.961392] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:50.962629] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224210961303, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:16:50.962651] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224215962649, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:16:50.996352] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5922-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:50.997081] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA05-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.005795] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3B1-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:51.108873] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C591C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.127256] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:51.128487] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224211118444, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:16:51.128512] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224216128510, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:16:51.128535] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224216128532, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:51.129857] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E8-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.132358] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E8-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:51.132378] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E8-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.141017] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0FD-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.153386] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF09E-0-0] [lt=22] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.162012] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B4-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.164565] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B4-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:51.164586] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B4-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.190100] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D55-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.190294] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7B-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.192797] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7B-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:51.192813] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7B-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.204283] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:51.204644] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA0A-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.205519] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224211204205, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:16:51.205544] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224216205542, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:16:51.206264] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:51.207451] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224211206191, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:16:51.207469] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224216207468, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:51.243629] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B6-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.246666] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B6-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.248577] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B6-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.248664] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B6-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.292074] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3906-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:51.296207] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3906-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:51.417555] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E928-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:51.418919] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD529D-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:51.420076] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E928-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:51.421437] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD529D-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:51.436468] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E56-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224211435662000, start_id=1755224211435662000, end_id=1755224211435662001) [2025-08-15 10:16:51.437311] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E56-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224211435662000, v:0}, tenant_id=1018) [2025-08-15 10:16:51.446139] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68B4-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.464138] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:51.464149] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:51.464153] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:51.481682] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A099B-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}) [2025-08-15 10:16:51.481723] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A099B-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:16:51.505862] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0FB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.505902] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E76A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.508566] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D3-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:51.510083] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD533C-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:51.511068] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D3-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:51.512573] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD533C-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:51.536095] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224209605949003, replayable_scn:1755224209605949003, readable_scn:1755224209605949003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224211533515, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224209605949003, replayable_scn:1755224209605949003, readable_scn:1755224209605949003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224211533515, ora_rowscn_=1755224210892919000, content_changed=true) [2025-08-15 10:16:51.536291] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:51.536307] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:51.536315] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:51.536886] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:51.536898] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:51.536904] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:51.542219] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:51.542235] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:51.542241] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:51.548362] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:51.548374] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:51.548379] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:51.553753] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:51.553764] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:51.553772] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:51.553782] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1006, cost_time_us=175, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:51.575186] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227811573910}) [2025-08-15 10:16:51.575207] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:51.575222] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:51.576293] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE2-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:51.578817] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE2-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:51.581434] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227811580188}) [2025-08-15 10:16:51.581450] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:51.581460] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:51.582694] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227811581472}) [2025-08-15 10:16:51.582709] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:51.582719] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:51.617051] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E6-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:51.619513] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:51.671175] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A55-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.671297] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAD-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.673740] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A55-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:51.673762] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A55-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.673838] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAD-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:51.673856] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAD-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.695056] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68B7-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.706242] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDC9-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.710620] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:51.710633] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:51.710638] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:51.755263] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDD8-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.755393] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF0A2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.795672] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709536][T1014_TimerWK3_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:51.830355] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA6-0-0] [lt=11] get number(ret=0, service_type_=0, range=1, base_id=1755224211829941000, start_id=1755224211829941000, end_id=1755224211829941001) [2025-08-15 10:16:51.831140] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA6-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224211829941000, v:0}, tenant_id=1014) [2025-08-15 10:16:51.832114] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7896-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.832129] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841407-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224211428315004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224211428315004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224211428315004, v:0}}], last_dump_ts_=1755224206800992, force_dump=false) [2025-08-15 10:16:51.832157] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841407-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.833265] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7728-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.833385] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5D-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.835164] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7896-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.835213] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841407-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224211428315004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224211428315004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224211428315004, v:0}}], last_dump_ts_=1755224206804185, force_dump=false) [2025-08-15 10:16:51.835238] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841407-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.836130] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7728-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.836312] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.837141] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7896-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.837235] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7896-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.837260] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841407-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.837357] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841407-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.838052] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7728-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.838138] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7728-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.838254] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.838331] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.839020] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.841916] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08F-0-0] [lt=16] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224211680679001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224211680679001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224211680679001, v:0}}], last_dump_ts_=1755224206812895, force_dump=false) [2025-08-15 10:16:51.841944] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08F-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.843871] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.844046] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC08F-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.852701] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:51.852957] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D58-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.853948] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224211852626, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:16:51.853972] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224216853970, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:16:51.865258] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8890-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.866550] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6F-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.867792] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8890-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:51.867810] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8890-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.869076] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6F-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:51.869094] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A6F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:51.871873] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475910][T1008_TimerWK2_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:16:51.892779] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC65F-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.896773] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7686-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.899652] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7686-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.901473] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7686-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.901552] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7686-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:51.951083] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224211550322, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224209426107003, replayable_scn:1755224209426107003, readable_scn:1755224209426107003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:51.996430] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5923-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:51.997155] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA06-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.007240] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3B2-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:52.108948] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C591D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.135094] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E9-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.137636] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E9-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:52.137654] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95E9-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.141062] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0FE-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.153461] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF09F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.167271] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B5-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.169840] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B5-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:52.169860] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B5-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.187513] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D8-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1015, thread_idx=0) [2025-08-15 10:16:52.188768] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1015, only_for_display=false) [2025-08-15 10:16:52.190016] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:52.190173] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D56-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.195560] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7C-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.198094] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7C-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:52.198110] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7C-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.203878] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.204718] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA0B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.205105] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.206284] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.206306] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1015, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:52.206313] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:52.206329] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=14] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.206339] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1015, ls_id:{id:1}, replicas:[{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:52.206460] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=95] build ls locality stat map(tenant_id=1015, ls_id={id:1}) [2025-08-15 10:16:52.206470] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.206484] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.206494] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.206514] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=14] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:52.206598] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=63] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.206614] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.206623] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.206628] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.206637] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3926, tenant_id=1015, ls_id={id:1}) [2025-08-15 10:16:52.206644] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:52.207789] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19021, tenant_id=1015, acc_dr_task=0) [2025-08-15 10:16:52.209031] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1016, only_for_display=false) [2025-08-15 10:16:52.210260] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:52.221398] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727957][T1018_TimerWK1_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=6] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:52.224145] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.225308] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=27] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.226461] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.226484] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:52.226501] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.226512] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1}, replicas:[{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:52.226615] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=79] build ls locality stat map(tenant_id=1016, ls_id={id:1}) [2025-08-15 10:16:52.226623] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.226636] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.226647] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.226659] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:52.226740] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.226754] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.226763] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.226771] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.226779] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3791, tenant_id=1016, ls_id={id:1}) [2025-08-15 10:16:52.226789] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:52.227876] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:52.248708] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.249757] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B7-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.249882] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.251056] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.251079] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:52.251098] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.251109] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1001}, replicas:[{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:52.251218] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=84] build ls locality stat map(tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:16:52.251227] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.251240] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.251251] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.251264] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:52.251343] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.251358] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.251368] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.251374] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.251395] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=16] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3844, tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:16:52.251408] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=10] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:52.252526] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:52.252799] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B7-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.254792] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B7-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.254880] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B7-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.273287] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.274427] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.275591] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.275616] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:52.275633] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.275644] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1002}, replicas:[{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:52.275746] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=80] build ls locality stat map(tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:16:52.275753] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.275766] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.275776] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.275788] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:52.275871] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.275885] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.275895] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.275900] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.275910] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3770, tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:16:52.275919] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:52.276197] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619487][T1012_TimerWK2_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:52.276986] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:52.293374] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95CE3-0-0] [lt=13] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}) [2025-08-15 10:16:52.293416] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95CE3-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:16:52.297768] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.298926] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.300076] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:52.300098] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:52.300113] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.300126] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1003}, replicas:[{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:52.300227] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=79] build ls locality stat map(tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:16:52.300234] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.300248] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.300259] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:52.300270] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:52.300347] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.300360] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.300369] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.300374] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:52.300382] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3764, tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:16:52.300393] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:52.301481] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D9-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93677, tenant_id=1016, acc_dr_task=0) [2025-08-15 10:16:52.301541] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3907-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:52.304026] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3907-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:52.335958] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487605][T1004_TimerWK2_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=27] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:52.350563] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:52.351833] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224212350480, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:16:52.351862] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224217351860, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:16:52.425375] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E929-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:52.426716] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD529E-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:52.427851] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E929-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:52.429174] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD529E-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:52.442104] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:52.443372] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224212442025, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:16:52.443398] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224217443396, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:16:52.446213] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68B5-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.464235] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:52.464252] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:52.464257] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:52.505936] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0FC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.505946] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E76B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.516390] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D4-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:52.516744] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF239-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.517779] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD533D-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:52.518891] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D4-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:52.520252] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD533D-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:52.536261] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:52.536281] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:52.536288] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:52.536932] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:52.536950] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:52.536958] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:52.538249] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:52.542300] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:52.542311] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:52.542317] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:52.543596] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:52.548419] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:52.548431] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:52.548437] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:52.549340] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D13A6-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81835938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82226562GB}) [2025-08-15 10:16:52.549378] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D13A6-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:16:52.549714] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:52.553831] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:52.553841] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:52.553849] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:52.584187] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE3-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:52.586685] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE3-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:52.626014] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E7-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:52.628762] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E7-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:52.676513] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A56-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.676582] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAE-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.679084] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A56-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:52.679102] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A56-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.679205] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAE-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:52.679224] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAE-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.695132] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68B8-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.706322] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDCA-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.710579] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:52.710597] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:52.710605] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:52.755335] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDD9-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.755470] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF0A3-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.815485] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:52.816747] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224212815417, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:16:52.816776] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224217816774, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:16:52.817975] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224212816622, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:16:52.817995] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224217817994, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:16:52.838255] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7897-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.838292] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841408-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.839091] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7729-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.839241] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.841300] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841408-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.841456] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7897-0-0] [lt=19] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224212429238004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224212429238004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224212429238004, v:0}}], last_dump_ts_=1755224207810544, force_dump=false) [2025-08-15 10:16:52.841479] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7897-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.842129] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7729-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.842370] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5E-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.843304] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841408-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.843398] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841408-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.843527] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7897-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.843609] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7897-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.844134] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7729-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.844237] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7729-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.844260] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.844363] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.844960] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC090-0-0] [lt=9] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224212429238004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224212429238004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224212429238004, v:0}}], last_dump_ts_=1755224207815885, force_dump=false) [2025-08-15 10:16:52.844984] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC090-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.847814] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC090-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.849705] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC090-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.849790] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC090-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.853008] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D59-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.870560] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8891-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.871728] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A70-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.873112] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8891-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:52.873130] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8891-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.874239] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A70-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:52.874259] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A70-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:52.902440] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7687-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.905431] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7687-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.907425] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7687-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.907510] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7687-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:52.978833] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:52.980108] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224212978760, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:16:52.980134] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224217980132, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:16:52.996505] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5924-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:52.997231] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA07-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.008656] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3B3-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:53.064890] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:53.066133] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224213064863, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:16:53.066158] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224218066156, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:16:53.109021] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C591E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.140357] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EA-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.141138] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0FF-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.142862] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EA-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:53.142879] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EA-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.153534] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF0A0-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.172625] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B6-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.175206] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B6-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:53.175229] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B6-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.190249] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D57-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.200867] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7D-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.203357] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7D-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:53.203375] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.204789] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA0C-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.255894] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B8-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.258774] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:340) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B8-0-0] [lt=15] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1014, ls_id:{id:1}, sync_scn:{val:1755224212930615002, v:0}, readable_scn:{val:1755224212930615002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-08-15 10:16:53.259131] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B8-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.261101] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B8-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.261197] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B8-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.261232] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:459) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B8-0-0] [lt=6] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1755224210927641003, v:0}) [2025-08-15 10:16:53.279157] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EF-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1013, thread_idx=0) [2025-08-15 10:16:53.280431] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1013, only_for_display=false) [2025-08-15 10:16:53.281668] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:53.295594] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.296755] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.297935] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.297964] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1013, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:53.297982] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.297992] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1013, ls_id:{id:1}, replicas:[{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:53.298096] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=81] build ls locality stat map(tenant_id=1013, ls_id={id:1}) [2025-08-15 10:16:53.298111] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.298125] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.298136] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.298144] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:53.298222] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.298236] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.298245] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.298251] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.298259] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3843, tenant_id=1013, ls_id={id:1}) [2025-08-15 10:16:53.298269] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:53.299341] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18910, tenant_id=1013, acc_dr_task=0) [2025-08-15 10:16:53.300560] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1014, only_for_display=false) [2025-08-15 10:16:53.301763] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:53.309388] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3908-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:53.311890] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3908-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:53.315558] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.316706] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.317882] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.317906] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:53.317922] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.317933] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1}, replicas:[{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:53.318043] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=86] build ls locality stat map(tenant_id=1014, ls_id={id:1}) [2025-08-15 10:16:53.318051] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.318064] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.318079] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=12] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.318091] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:53.318169] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.318183] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.318192] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.318202] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.318209] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3805, tenant_id=1014, ls_id={id:1}) [2025-08-15 10:16:53.318219] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:53.319303] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:53.340078] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.341229] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.342379] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.342403] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:53.342420] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.342430] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1001}, replicas:[{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:53.342539] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=85] build ls locality stat map(tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:16:53.342547] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.342564] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.342571] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.342583] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:53.342659] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.342674] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.342683] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.342692] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.342698] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3750, tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:16:53.342708] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:53.343821] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:53.364571] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.365726] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.366874] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.366894] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:53.366909] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.366918] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1002}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:53.367026] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=85] build ls locality stat map(tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:16:53.367034] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.367045] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.367053] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.367060] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:53.367135] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.367147] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.367154] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.367161] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.367168] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3753, tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:16:53.367176] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:53.368244] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:53.389251] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.390398] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.391550] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:53.391574] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:53.391586] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.391597] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1003}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:53.391705] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=84] build ls locality stat map(tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:16:53.391714] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.391727] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.391734] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:53.391745] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:53.391823] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.391837] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.391847] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.391853] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:53.391861] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3749, tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:16:53.391871] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:53.393012] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69F0-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93655, tenant_id=1014, acc_dr_task=0) [2025-08-15 10:16:53.433212] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E92A-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:53.434524] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD529F-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:53.435717] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E92A-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:53.438182] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD529F-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:53.446287] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68B6-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.464161] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224211928806004, replayable_scn:1755224211928806004, readable_scn:1755224211928806004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224213460527, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224211928806004, replayable_scn:1755224211928806004, readable_scn:1755224211928806004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224213460527, ora_rowscn_=1755224213257548000, content_changed=true) [2025-08-15 10:16:53.464344] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:53.464356] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:53.464363] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:53.482183] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A099E-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.93164062GB}) [2025-08-15 10:16:53.482228] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A099E-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=50, ddl_arg=NULL) [2025-08-15 10:16:53.506011] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0FD-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.506015] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E76C-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.524371] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D5-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:53.525569] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD533E-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:53.526898] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D5-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:53.528078] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD533E-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:53.531930] INFO [RS] prepare (ob_mview_maintenance_task.cpp:176) [2619488][T1012_TimerWK3_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=9] mview maintenance task prepare success(tenant_id_=1012, round_=77508, prepare_cost_us_=1) [2025-08-15 10:16:53.536271] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:53.536282] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:53.536287] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:53.536826] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224211428315004, replayable_scn:1755224211428315004, readable_scn:1755224211428315004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224213534229, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224211428315004, replayable_scn:1755224211428315004, readable_scn:1755224211428315004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224213534229, ora_rowscn_=1755224212840114000, content_changed=true) [2025-08-15 10:16:53.536999] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:53.537016] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:53.537021] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:53.542194] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224211436714004, replayable_scn:1755224211436714004, readable_scn:1755224211436714004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224213539464, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224211436714004, replayable_scn:1755224211436714004, readable_scn:1755224211436714004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224213539464, ora_rowscn_=1755224212841169000, content_changed=true) [2025-08-15 10:16:53.542365] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:53.542375] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:53.542380] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:53.548285] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224211428315004, replayable_scn:1755224211428315004, readable_scn:1755224211428315004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224213544717, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224211428315004, replayable_scn:1755224211428315004, readable_scn:1755224211428315004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224213544717, ora_rowscn_=1755224212841169000, content_changed=true) [2025-08-15 10:16:53.548469] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:53.548488] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:53.548502] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:53.553771] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224211680679001, replayable_scn:1755224211680679001, readable_scn:1755224211680679001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224213551037, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224211680679001, replayable_scn:1755224211680679001, readable_scn:1755224211680679001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224213551037, ora_rowscn_=1755224212846440000, content_changed=true) [2025-08-15 10:16:53.553961] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:53.553974] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:53.553979] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:53.592668] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE4-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:53.597481] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE4-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:53.622573] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:53.623817] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224213622492, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:16:53.623841] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224218623839, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:16:53.634138] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E8-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:53.636693] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E8-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:53.681809] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A57-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.681929] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAF-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.684347] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A57-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:53.684364] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A57-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.684487] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAF-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:53.684507] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFAF-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.695206] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68B9-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.706397] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDCB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.710432] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224211428315004, replayable_scn:1755224211428315004, readable_scn:1755224211428315004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224213707599, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224211428315004, replayable_scn:1755224211428315004, readable_scn:1755224211428315004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224213707599, ora_rowscn_=1755224212840114000, content_changed=true) [2025-08-15 10:16:53.710622] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:53.710640] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:53.710652] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:53.755409] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDDA-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.755550] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF0A4-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.844398] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841409-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.844656] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7898-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.845244] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772A-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224213431076004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224213431076004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224213431076004, v:0}}], last_dump_ts_=1755224208814635, force_dump=false) [2025-08-15 10:16:53.845268] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772A-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.845343] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5F-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.847318] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841409-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.847631] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7898-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.848341] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.848432] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772A-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224213431076004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224213431076004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224213431076004, v:0}}], last_dump_ts_=1755224208817827, force_dump=false) [2025-08-15 10:16:53.848454] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772A-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.849220] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841409-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.849299] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841409-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.849629] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7898-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.849716] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7898-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.850334] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.850436] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.850468] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C5F-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.850537] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.850699] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC091-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.853079] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D5A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.853703] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC091-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.855736] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC091-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.855833] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC091-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.875833] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8892-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.876895] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A71-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.878364] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8892-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:53.878382] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8892-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.879401] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A71-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:53.879419] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A71-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:53.897699] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC660-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.908538] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7688-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.911808] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7688-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.913787] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7688-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.913890] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7688-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:53.996579] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5925-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:53.997304] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA08-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.010083] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3B4-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:54.109097] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C591F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.141208] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF100-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.145630] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EB-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.148126] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EB-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:54.148145] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EB-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.153608] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF0A1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.177975] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B7-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.180561] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B7-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:54.180578] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B7-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.190323] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D58-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.204861] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA0D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.206194] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7E-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.208760] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7E-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:54.208779] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7E-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.262261] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B9-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.265416] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B9-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224213931531003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224213931531003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224213931531003, v:0}}], last_dump_ts_=1755224209233761, force_dump=false) [2025-08-15 10:16:54.265442] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B9-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.267478] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B9-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.267576] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83B9-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.293878] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D91-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}) [2025-08-15 10:16:54.293919] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D91-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:16:54.317176] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3909-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:54.319648] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3909-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:54.441055] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E92B-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:54.443418] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD52A0-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:54.443534] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E92B-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:54.445910] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD52A0-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:54.446361] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68B7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.464426] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:54.464438] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:54.464444] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:54.465736] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:54.506085] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0FE-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.506086] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E76D-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.517004] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF23A-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.532546] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D6-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:54.533422] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD533F-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:54.535127] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D6-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:54.536201] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224212596111002, replayable_scn:1755224212596111002, readable_scn:1755224212596111002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224214533211, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224212596111002, replayable_scn:1755224212596111002, readable_scn:1755224212596111002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224214533211, ora_rowscn_=1755224213910538001, content_changed=true) [2025-08-15 10:16:54.536381] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:54.536393] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:54.536404] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:54.536986] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:54.537001] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:54.537007] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:54.537154] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD533F-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:54.537671] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:54.542442] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:54.542453] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:54.542462] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:54.548510] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:54.548525] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:54.548537] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:54.549775] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAD7-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82226562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}) [2025-08-15 10:16:54.549814] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAD7-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:16:54.553952] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:54.553967] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:54.553978] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:54.555256] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:54.587010] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227814585839}) [2025-08-15 10:16:54.587026] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:54.587037] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:54.592913] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227814591751}) [2025-08-15 10:16:54.592928] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:54.592935] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:54.594112] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227814592953}) [2025-08-15 10:16:54.594125] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:54.594135] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:54.602761] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE5-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:54.605237] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE5-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:54.642029] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E9-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:54.643050] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:54.644300] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224214642975, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:16:54.644325] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224219644323, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:16:54.644466] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:104) [17458][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=10] start checking lost replicas [2025-08-15 10:16:54.645725] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62E9-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:54.687075] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A58-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.687243] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB0-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.689594] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A58-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:54.689610] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A58-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.689790] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB0-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:54.689806] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB0-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.695281] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68BA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.706462] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDCC-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.710662] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:54.710671] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:54.710677] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:54.710686] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_time_us=185, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:54.711954] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:54.755480] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDDB-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.755621] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF0A5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.850318] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.850700] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7899-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224214767700001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224214767700001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224214767700001, v:0}}], last_dump_ts_=1755224209819804, force_dump=false) [2025-08-15 10:16:54.850718] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7899-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.851384] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C60-0-0] [lt=9] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224214432165004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224214432165004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224214432165004, v:0}}], last_dump_ts_=1755224209821172, force_dump=false) [2025-08-15 10:16:54.851406] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C60-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.851465] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772B-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.853154] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D5B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.853474] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140A-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.853700] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7899-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.854378] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C60-0-0] [lt=8] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224214432165004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224214432165003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224214432165003, v:0}}], last_dump_ts_=1755224209824239, force_dump=false) [2025-08-15 10:16:54.854406] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C60-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.854417] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772B-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.855637] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.855706] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7899-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.855794] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7899-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.855810] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.856371] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.856464] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.856494] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C60-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.856592] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C60-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.856749] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC092-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.859689] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC092-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.861583] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC092-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.861702] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC092-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.881098] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8893-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.882088] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A72-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.883696] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8893-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:54.883712] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8893-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.884578] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A72-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:54.884594] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A72-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:54.914954] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7689-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224214659332000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224214740397000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224214740397000, v:0}}], last_dump_ts_=1755224209884899, force_dump=false) [2025-08-15 10:16:54.914982] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7689-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.918754] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7689-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224214740397000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224214740397001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224214740397001, v:0}}], last_dump_ts_=1755224209887842, force_dump=false) [2025-08-15 10:16:54.918780] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7689-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.920904] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7689-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.921025] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7689-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:54.952712] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=17] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224214551075, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224212429238004, replayable_scn:1755224212429238004, readable_scn:1755224212429238004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:54.996648] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5926-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:54.997382] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA09-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.011490] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3B5-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:55.019678] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=5] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:55.109171] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5920-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.141280] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF101-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.150842] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EC-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.153378] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EC-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:55.153396] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EC-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.153679] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF0A2-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.183316] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B8-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.185954] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B8-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:55.185972] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B8-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.190396] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D59-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.204930] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA0E-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.211522] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7F-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.214069] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7F-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:55.214087] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB7F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.268562] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BA-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224214932775003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224215239899000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224215239899000, v:0}}], last_dump_ts_=1755224210236978, force_dump=false) [2025-08-15 10:16:55.268586] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BA-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.271584] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BA-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.273639] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BA-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.273752] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BA-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.289994] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E57-0-0] [lt=13] start do ObArchiveSchedulerService round(round=101516) [2025-08-15 10:16:55.293553] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E57-0-0] [lt=11][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:16:55.293568] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E57-0-0] [lt=11][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1014 AND dest_no = 0 AND name = 'state', exec_tenant_id=1013) [2025-08-15 10:16:55.293609] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E57-0-0] [lt=7][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1014, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:16:55.293622] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E57-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1014, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:16:55.293631] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E57-0-0] [lt=7][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1014, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:16:55.325012] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B390A-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:55.327503] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B390A-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:55.446387] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68B8-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.448936] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E92C-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:55.451339] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD52A1-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:55.451453] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E92C-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:55.453908] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD52A1-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:55.463858] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:55.463869] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:55.463877] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:55.482677] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98DA7-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.93164062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}) [2025-08-15 10:16:55.482724] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98DA7-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=52, ddl_arg=NULL) [2025-08-15 10:16:55.506123] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0FF-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.506158] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E76E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.536474] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:55.536491] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:55.536493] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:55.536498] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:55.536507] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:55.536514] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:55.541065] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D7-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:55.542390] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5340-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:55.542553] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:55.542566] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:55.542573] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:55.543656] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D7-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:55.544954] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5340-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:55.548586] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:55.548597] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:55.548603] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:55.554079] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:55.554092] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:55.554098] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:55.610548] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE6-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:55.613039] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:55.638196] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:55.639448] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224215638119, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:16:55.639474] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224220639472, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:16:55.651045] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62EA-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:55.653601] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62EA-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:55.691915] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB1-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.692828] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A59-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.694473] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB1-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:55.694494] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB1-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.695353] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68BB-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.695913] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A59-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:55.695932] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A59-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.706536] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDCD-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.710709] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:55.710723] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:55.710730] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:55.741542] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5D-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1011, thread_idx=0) [2025-08-15 10:16:55.742786] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1011, only_for_display=false) [2025-08-15 10:16:55.744007] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:55.755558] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDDC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.755697] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF0A6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.759177] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.760327] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.761468] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.761489] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1011, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:55.761503] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.761513] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1011, ls_id:{id:1}, replicas:[{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:55.761623] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=85] build ls locality stat map(tenant_id=1011, ls_id={id:1}) [2025-08-15 10:16:55.761631] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.761644] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.761653] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.761661] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:55.761739] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.761752] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.761760] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.761768] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.761775] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3761, tenant_id=1011, ls_id={id:1}) [2025-08-15 10:16:55.761784] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:55.763299] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=20514, tenant_id=1011, acc_dr_task=0) [2025-08-15 10:16:55.764521] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1012, only_for_display=false) [2025-08-15 10:16:55.765684] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438022][T1006_TimerWK1_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:55.765738] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:55.780914] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.782077] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.783220] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.783243] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:55.783256] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.783266] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1}, replicas:[{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:55.783374] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=83] build ls locality stat map(tenant_id=1012, ls_id={id:1}) [2025-08-15 10:16:55.783382] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.783394] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.783403] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.783413] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:55.783491] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.783504] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.783511] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.783519] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.783526] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3779, tenant_id=1012, ls_id={id:1}) [2025-08-15 10:16:55.783534] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:55.785044] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:55.807232] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.808390] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.809542] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.809561] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=14] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:55.809575] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.809587] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1001}, replicas:[{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:55.809699] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=88] build ls locality stat map(tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:16:55.809708] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.809721] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.809731] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.809744] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:55.809823] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.809833] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.809842] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.809848] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.809857] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3786, tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:16:55.809865] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:55.811376] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:55.833441] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.834592] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.835728] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.835751] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:55.835769] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.835780] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1002}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:55.835890] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=85] build ls locality stat map(tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:16:55.835899] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.835913] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.835920] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.835932] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:55.836018] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=63] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.836036] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.836046] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.836053] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.836061] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3759, tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:16:55.836072] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:55.837592] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:55.853225] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D5C-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.856777] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.856814] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.857468] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.857600] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C61-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.859843] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.859888] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140B-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.859928] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789A-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.860482] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772C-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.860713] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C61-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.861015] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.861844] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789A-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.861886] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.861981] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.862026] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.862248] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:55.862270] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:55.862288] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.862375] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.862299] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1003}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:55.862409] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=85] build ls locality stat map(tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:16:55.862419] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.862431] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.862440] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:55.862451] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.862447] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:55.862525] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.862540] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.862549] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.862554] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:55.862561] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3879, tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:16:55.862570] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:55.862669] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC093-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.862758] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C61-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.862863] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C61-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.864197] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C5E-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=100884, tenant_id=1012, acc_dr_task=0) [2025-08-15 10:16:55.865582] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC093-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.867467] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC093-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.867565] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC093-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.886452] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8894-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.887287] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A73-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.888971] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8894-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:55.888988] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8894-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.889802] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A73-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:55.889820] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A73-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:55.902615] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC661-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.921980] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768A-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.925171] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768A-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.927172] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.927272] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:55.962724] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:55.963985] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224215962649, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:16:55.964009] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224220964008, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:16:55.996720] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5927-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:55.997416] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA0A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.012883] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3B6-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:56.018278] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDB-0-0] [lt=13] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1007, thread_idx=0) [2025-08-15 10:16:56.019623] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=21] start try tenant disaster recovery(tenant_id=1007, only_for_display=false) [2025-08-15 10:16:56.020851] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:56.035126] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.036283] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.037454] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.037476] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1007, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:56.037489] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.037501] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1007, ls_id:{id:1}, replicas:[{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:56.037618] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=91] build ls locality stat map(tenant_id=1007, ls_id={id:1}) [2025-08-15 10:16:56.037626] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.037637] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.037646] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.037655] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:56.037731] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.037744] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.037754] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.037764] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.037773] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3820, tenant_id=1007, ls_id={id:1}) [2025-08-15 10:16:56.037784] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:56.038849] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19226, tenant_id=1007, acc_dr_task=0) [2025-08-15 10:16:56.040091] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1008, only_for_display=false) [2025-08-15 10:16:56.041313] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:56.055216] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.056370] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.057519] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.057542] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:56.057559] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.057571] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1}, replicas:[{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:56.057674] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=80] build ls locality stat map(tenant_id=1008, ls_id={id:1}) [2025-08-15 10:16:56.057683] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.057700] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.057715] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=12] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.057727] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:56.057806] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.057823] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.057833] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.057838] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.057846] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3784, tenant_id=1008, ls_id={id:1}) [2025-08-15 10:16:56.057852] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:56.058929] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:56.079772] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.080969] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.082119] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.082140] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:56.082156] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.082166] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1001}, replicas:[{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:56.082266] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=78] build ls locality stat map(tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:16:56.082274] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.082286] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.082295] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.082304] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:56.082379] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=54] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.082391] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.082398] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.082405] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.082412] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3796, tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:16:56.082419] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:56.083481] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:56.105439] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.107081] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.108298] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.108328] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:56.108350] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.108362] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1002}, replicas:[{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:56.108547] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=145] build ls locality stat map(tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:16:56.108564] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.108579] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.108593] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=11] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.108608] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:56.108754] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=109] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.108773] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.108784] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.108795] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.108806] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=4559, tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:16:56.108818] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:56.109245] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5921-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.110132] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:56.131774] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.132923] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.134075] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:56.134096] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:56.134111] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.134121] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1003}, replicas:[{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:56.134223] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=80] build ls locality stat map(tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:16:56.134231] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.134242] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.134251] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:56.134260] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:56.134337] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.134346] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.134354] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.134360] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:56.134367] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3736, tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:16:56.134375] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:56.135474] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7EDC-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=96613, tenant_id=1008, acc_dr_task=0) [2025-08-15 10:16:56.137353] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:56.138618] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224216128510, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:16:56.138641] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224221138640, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:16:56.138659] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224221138657, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:56.141355] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF102-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.142805] INFO [RS.LB] idle (ob_root_balancer.cpp:87) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=30] root balance idle(idle_time=10000000) [2025-08-15 10:16:56.142824] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:70) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=14] start check unit migrate finish [2025-08-15 10:16:56.142842] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=10] try check migrate unit finish by tenant(tenant_id=1) [2025-08-15 10:16:56.152298] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1004) [2025-08-15 10:16:56.153759] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF0A3-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.156113] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95ED-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.158658] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95ED-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:56.158678] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95ED-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.178287] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=19] try check migrate unit finish by tenant(tenant_id=1006) [2025-08-15 10:16:56.188691] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B9-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.190473] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D5A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.191235] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B9-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:56.191254] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88B9-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.203972] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1008) [2025-08-15 10:16:56.205010] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA0F-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.205616] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:56.206848] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224216205542, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:16:56.206872] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224221206870, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:16:56.207533] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:56.208753] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=6] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224216207468, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:16:56.208774] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224221208773, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:16:56.216861] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB80-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.219412] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB80-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:56.219432] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB80-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.229710] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1012) [2025-08-15 10:16:56.255400] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1014) [2025-08-15 10:16:56.274758] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BB-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.277818] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BB-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.280003] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BB-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.280163] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BB-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.280972] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1016) [2025-08-15 10:16:56.294392] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA45-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}) [2025-08-15 10:16:56.294431] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA45-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:16:56.306565] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1018) [2025-08-15 10:16:56.332146] INFO [RS] check (ob_alter_locality_finish_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D2-0-0] [lt=15] start to check alter locality finish [2025-08-15 10:16:56.332826] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B390B-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:56.333289] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D3-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.334444] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D3-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.335461] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B390B-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:56.336739] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D5-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.337870] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D5-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.340139] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D7-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.341267] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D7-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.343526] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D9-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.344662] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815D9-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.346951] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815DB-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.348078] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815DB-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.350328] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815DD-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.351455] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815DD-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.353743] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815DF-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.354894] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815DF-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.357158] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E1-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.358299] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E1-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.359462] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E1-0-0] [lt=12] start check shrink resource pool [2025-08-15 10:16:56.359479] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E2-0-0] [lt=12] start check shrink resource pool(tenant_id=1) [2025-08-15 10:16:56.360583] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E2-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.361738] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E2-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.361789] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E3-0-0] [lt=12] start check shrink resource pool(tenant_id=1004) [2025-08-15 10:16:56.362900] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E3-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.364030] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E3-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.364080] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E4-0-0] [lt=13] start check shrink resource pool(tenant_id=1006) [2025-08-15 10:16:56.365198] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E4-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.366354] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E4-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.366407] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E5-0-0] [lt=12] start check shrink resource pool(tenant_id=1008) [2025-08-15 10:16:56.367517] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E5-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.368674] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E5-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.368723] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E6-0-0] [lt=12] start check shrink resource pool(tenant_id=1012) [2025-08-15 10:16:56.369819] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E6-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.370953] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E6-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.371004] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E7-0-0] [lt=12] start check shrink resource pool(tenant_id=1014) [2025-08-15 10:16:56.372107] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E7-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.373246] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E7-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.373294] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E8-0-0] [lt=12] start check shrink resource pool(tenant_id=1016) [2025-08-15 10:16:56.374393] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E8-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.375554] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E8-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.375604] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E9-0-0] [lt=12] start check shrink resource pool(tenant_id=1018) [2025-08-15 10:16:56.376729] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E9-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.377900] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E9-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.377954] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:99) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E9-0-0] [lt=10] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.377966] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815E9-0-0] [lt=8] start check primary zone [2025-08-15 10:16:56.377976] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815EA-0-0] [lt=6] start check primary zone(tenant_id=1) [2025-08-15 10:16:56.379082] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815EA-0-0] [lt=4][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.379129] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815EB-0-0] [lt=11] start check primary zone(tenant_id=1004) [2025-08-15 10:16:56.380266] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815EB-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.380314] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815EC-0-0] [lt=11] start check primary zone(tenant_id=1006) [2025-08-15 10:16:56.381439] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815EC-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.381488] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815ED-0-0] [lt=12] start check primary zone(tenant_id=1008) [2025-08-15 10:16:56.382596] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815ED-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.382646] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815EE-0-0] [lt=13] start check primary zone(tenant_id=1012) [2025-08-15 10:16:56.383752] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815EE-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.383801] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815EF-0-0] [lt=11] start check primary zone(tenant_id=1014) [2025-08-15 10:16:56.384931] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815EF-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.385001] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F0-0-0] [lt=18] start check primary zone(tenant_id=1016) [2025-08-15 10:16:56.386134] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F0-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.386183] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=12] start check primary zone(tenant_id=1018) [2025-08-15 10:16:56.387309] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:16:56.387360] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:166) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=13] start do unit balance [2025-08-15 10:16:56.393645] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:350) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=8] start distribute for server status change [2025-08-15 10:16:56.393742] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:401) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=11] finish distribute for server status change(ret=0) [2025-08-15 10:16:56.393753] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:200) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=9] finish do unit balance(ret=0) [2025-08-15 10:16:56.399502] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:115) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=4] start to do tenant group unit balance [2025-08-15 10:16:56.399542] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=12] start to do tenant group unit balance(zone=zone1) [2025-08-15 10:16:56.399556] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=8] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.203.1.34:2882", disk_in_use_:3089104896, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:56.399575] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=13] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:56.399602] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=12] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:56.399619] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=9] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:56.399630] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone1) [2025-08-15 10:16:56.399640] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=6] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:56.399660] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=13] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.1.34:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:56.399702] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=33] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.203.1.34:2882", disk_in_use_:3089104896, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:56.399710] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=6] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.1.34:2882", disk_in_use_:3089104896, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:56.399723] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=9] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:56.399740] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=12] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1006, unit_config:{unit_config_id:1005, name:"config_opsone_zone1_s_2_4_ywx", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7988, unit_config:0x7f1e9aa71c38, pool:0x7f1c3beb2f18}, {unit_id:1008, tenant_id:1006, resource_pool_id:1008, unit_config:{unit_config_id:1008, name:"config_test_zone1_s_2_4_emp", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f77d8, unit_config:0x7f1e9aa71998, pool:0x7f1c3beb27a8}, {unit_id:1011, tenant_id:1008, resource_pool_id:1011, unit_config:{unit_config_id:1011, name:"config_mcp_zone1_s_4_8_tmc", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7550, unit_config:0x7f1e9aa716f8, pool:0x7f1e443fda40}, {unit_id:1016, tenant_id:1012, resource_pool_id:1016, unit_config:{unit_config_id:1017, name:"config_oms_zone1_s_4_8_gzp", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f73a0, unit_config:0x7f1e9aa71458, pool:0x7f1e443fd2d0}, {unit_id:1019, tenant_id:1014, resource_pool_id:1019, unit_config:{unit_config_id:1019, name:"config_grafana_test_zone1_s_1_2_pri", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7118, unit_config:0x7f1e9aa71298, pool:0x7f1e443fc7a8}, {unit_id:1024, tenant_id:1016, resource_pool_id:1024, unit_config:{unit_config_id:1023, name:"config_temporalio_zone1_s_2_4_ykw", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6ce0, unit_config:0x7f1e9aa70f18, pool:0x7f1e74dff2d0}, {unit_id:1026, tenant_id:1018, resource_pool_id:1026, unit_config:{unit_config_id:1027, name:"config_media_zone1_s_2_4_kmn", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6b30, unit_config:0x7f1e9aa70b98, pool:0x7f1e74dfeb60}], not_grant_units=[], available_servers=["10.203.1.34:2882"]) [2025-08-15 10:16:56.399799] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=36] start to do tenant group unit balance(zone=zone2) [2025-08-15 10:16:56.399808] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=7] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.203.4.38:2882", disk_in_use_:3139436544, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:56.399820] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:56.399838] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=9] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:56.399850] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=7] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:56.399858] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone2) [2025-08-15 10:16:56.399863] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:56.399876] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.38:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:56.399912] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=28] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.203.4.38:2882", disk_in_use_:3139436544, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:56.399917] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=5] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.38:2882", disk_in_use_:3139436544, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:56.399927] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=7] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:56.399943] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=13] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1004, unit_config:{unit_config_id:1004, name:"config_opsone_zone2_s_2_4_wag", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7b38, unit_config:0x7f1e9aa71d18, pool:0x7f1c3beb3688}, {unit_id:1009, tenant_id:1006, resource_pool_id:1009, unit_config:{unit_config_id:1009, name:"config_test_zone2_s_2_4_yey", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7700, unit_config:0x7f1e9aa718b8, pool:0x7f1c3beb23f0}, {unit_id:1010, tenant_id:1008, resource_pool_id:1010, unit_config:{unit_config_id:1010, name:"config_mcp_zone2_s_4_8_bjb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7628, unit_config:0x7f1e9aa717d8, pool:0x7f1c3beb2038}, {unit_id:1018, tenant_id:1012, resource_pool_id:1018, unit_config:{unit_config_id:1016, name:"config_oms_zone2_s_4_8_kbx", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f71f0, unit_config:0x7f1e9aa71538, pool:0x7f1e443fcb60}, {unit_id:1021, tenant_id:1014, resource_pool_id:1021, unit_config:{unit_config_id:1021, name:"config_grafana_test_zone2_s_1_2_yyc", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f6f68, unit_config:0x7f1e9aa710d8, pool:0x7f1e443fc038}, {unit_id:1023, tenant_id:1016, resource_pool_id:1023, unit_config:{unit_config_id:1022, name:"config_temporalio_zone2_s_2_4_lsv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6db8, unit_config:0x7f1e9aa70ff8, pool:0x7f1e74dff688}, {unit_id:1027, tenant_id:1018, resource_pool_id:1027, unit_config:{unit_config_id:1025, name:"config_media_zone2_s_2_4_yzd", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6a58, unit_config:0x7f1e9aa70d58, pool:0x7f1e74dfe7a8}], not_grant_units=[], available_servers=["10.203.4.38:2882"]) [2025-08-15 10:16:56.399994] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=31] start to do tenant group unit balance(zone=zone3) [2025-08-15 10:16:56.400003] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.203.4.39:2882", disk_in_use_:3034578944, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:16:56.400015] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:16:56.400031] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=9] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:56.400043] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=7] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:56.400052] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone3) [2025-08-15 10:16:56.400056] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:56.400069] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=8] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.39:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:16:56.400100] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=24] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.203.4.39:2882", disk_in_use_:3034578944, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:16:56.400105] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=5] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.39:2882", disk_in_use_:3034578944, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:16:56.400115] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=7] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:16:56.400126] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A815F1-0-0] [lt=8] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1005, unit_config:{unit_config_id:1006, name:"config_opsone_zone3_s_2_4_vxv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7a60, unit_config:0x7f1e9aa71b58, pool:0x7f1c3beb32d0}, {unit_id:1007, tenant_id:1006, resource_pool_id:1007, unit_config:{unit_config_id:1007, name:"config_test_zone3_s_2_4_mui", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f78b0, unit_config:0x7f1e9aa71a78, pool:0x7f1c3beb2b60}, {unit_id:1012, tenant_id:1008, resource_pool_id:1012, unit_config:{unit_config_id:1012, name:"config_mcp_zone3_s_4_8_seu", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7478, unit_config:0x7f1e9aa71618, pool:0x7f1e443fd688}, {unit_id:1017, tenant_id:1012, resource_pool_id:1017, unit_config:{unit_config_id:1018, name:"config_oms_zone3_s_4_8_nvb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f72c8, unit_config:0x7f1e9aa71378, pool:0x7f1e443fcf18}, {unit_id:1020, tenant_id:1014, resource_pool_id:1020, unit_config:{unit_config_id:1020, name:"config_grafana_test_zone3_s_1_2_xbz", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7040, unit_config:0x7f1e9aa711b8, pool:0x7f1e443fc3f0}, {unit_id:1022, tenant_id:1016, resource_pool_id:1022, unit_config:{unit_config_id:1024, name:"config_temporalio_zone3_s_2_4_rxz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6e90, unit_config:0x7f1e9aa70e38, pool:0x7f1e74dffa40}, {unit_id:1025, tenant_id:1018, resource_pool_id:1025, unit_config:{unit_config_id:1026, name:"config_media_zone3_s_2_4_ukz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6c08, unit_config:0x7f1e9aa70c78, pool:0x7f1e74dfef18}], not_grant_units=[], available_servers=["10.203.4.39:2882"]) [2025-08-15 10:16:56.446462] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68B9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.456823] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E92D-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:56.459203] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD52A2-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:56.459323] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E92D-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:56.461682] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD52A2-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:56.464300] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224214932775004, replayable_scn:1755224214932775004, readable_scn:1755224214932775004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224216461544, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224214932775004, replayable_scn:1755224214932775004, readable_scn:1755224214932775004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224216461544, ora_rowscn_=1755224216276354000, content_changed=true) [2025-08-15 10:16:56.464469] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:56.464480] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:56.464490] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:56.506194] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF100-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.506232] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E76F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.517270] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF23B-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.536505] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:56.536517] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:56.536522] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:56.536932] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224214432165005, replayable_scn:1755224214432165005, readable_scn:1755224214432165005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224216534072, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224214432165005, replayable_scn:1755224214432165005, readable_scn:1755224214432165005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224216534072, ora_rowscn_=1755224215858925000, content_changed=true) [2025-08-15 10:16:56.537107] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=24] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:56.537120] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:56.537125] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:56.542427] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224214433222003, replayable_scn:1755224214433222003, readable_scn:1755224214433222003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224216540379, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224214433222003, replayable_scn:1755224214433222003, readable_scn:1755224214433222003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224216540379, ora_rowscn_=1755224215858925000, content_changed=true) [2025-08-15 10:16:56.542604] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:56.542614] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:56.542622] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:56.548460] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224214432165004, replayable_scn:1755224214432165004, readable_scn:1755224214432165003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224216545645, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224214432165004, replayable_scn:1755224214432165004, readable_scn:1755224214432165003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224216545645, ora_rowscn_=1755224215859980000, content_changed=true) [2025-08-15 10:16:56.548645] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:56.548656] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:56.548664] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:56.549056] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D8-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:56.550238] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAD8-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83007812GB}) [2025-08-15 10:16:56.550262] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5341-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:56.550278] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBAD8-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:16:56.551567] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D8-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:56.552768] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5341-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:56.553894] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224214432165004, replayable_scn:1755224214432165004, readable_scn:1755224214432165004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224216550911, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224214432165004, replayable_scn:1755224214432165004, readable_scn:1755224214432165004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224216550911, ora_rowscn_=1755224215864195000, content_changed=true) [2025-08-15 10:16:56.554073] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:56.554085] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:56.554090] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:56.579918] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487710][T1003_DRService][T1003][YB420ACB0122-0006337FE6577D87-0-0] [lt=13] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1003, thread_idx=1) [2025-08-15 10:16:56.583574] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [1487710][T1003_DRService][T1003][YB420ACB0122-0006337FE6577D88-0-0] [lt=13] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2376, tenant_id=1003, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:56.585927] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [1487710][T1003_DRService][T1003][YB420ACB0122-0006337FE6577D88-0-0] [lt=12] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2336, tenant_id=1004, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:16:56.618399] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE7-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:56.620929] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE7-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:56.659025] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62EB-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:56.661573] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62EB-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:56.695430] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68BC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.697239] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB2-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.698598] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A5A-0-0] [lt=37] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.699931] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB2-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:56.699955] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB2-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.701124] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A5A-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:56.701139] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A5A-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.706611] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDCE-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.710576] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224214768754000, replayable_scn:1755224214768754000, readable_scn:1755224214768754000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224216707530, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224214768754000, replayable_scn:1755224214768754000, readable_scn:1755224214768754000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224216707530, ora_rowscn_=1755224215858925000, content_changed=true) [2025-08-15 10:16:56.710749] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:56.710761] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:56.710766] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:56.755631] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDDD-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.755766] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF0A7-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.797010] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:56.853295] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D5D-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.862929] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140C-0-0] [lt=11] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224216435211005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224216435211005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224216435211005, v:0}}], last_dump_ts_=1755224211832128, force_dump=false) [2025-08-15 10:16:56.862959] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140C-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.862999] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789B-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.863363] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:56.863382] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.863870] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C62-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.864625] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224216853970, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:16:56.864654] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224221864651, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:16:56.865992] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140C-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224216435211005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224216435211005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224216435211005, v:0}}], last_dump_ts_=1755224211835212, force_dump=false) [2025-08-15 10:16:56.866002] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789B-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.866017] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.866349] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772D-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.866689] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C62-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.867931] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.868035] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140C-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.868091] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789B-0-0] [lt=25] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.868120] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.868337] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.868444] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.868605] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C62-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.868625] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC094-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.868686] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C62-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.871566] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC094-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224216683409001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224216683409000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224216683409001, v:0}}], last_dump_ts_=1755224211841916, force_dump=false) [2025-08-15 10:16:56.871589] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC094-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.873421] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC094-0-0] [lt=3] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.873496] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC094-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.877283] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=42] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:56.891676] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8895-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.892498] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A74-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.894221] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8895-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:56.894238] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8895-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.895015] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A74-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:56.895031] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A74-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:56.911773] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EABA-0-0] [lt=13] [RESTORE] restore scheduler quit [2025-08-15 10:16:56.913111] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EABA-0-0] [lt=11] get targets(ret=0, jobs=[], exec_tenant_id=1015, sql=select * from __all_recover_table_job) [2025-08-15 10:16:56.914331] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EABB-0-0] [lt=15] get targets(ret=0, jobs=[], exec_tenant_id=1015, sql=select * from __all_import_table_job) [2025-08-15 10:16:56.914352] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EABC-0-0] [lt=12][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224216914348, ref_timestamp=1755224156909958, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:16:56.914439] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EABD-0-0] [lt=9] [RESTORE] restore scheduler start [2025-08-15 10:16:56.914448] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EABE-0-0] [lt=7] [RESTORE] try process restore job [2025-08-15 10:16:56.915626] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EABE-0-0] [lt=3] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:16:56.928266] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768B-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.931400] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768B-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.933362] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.933448] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:56.996794] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5928-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:56.997493] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA0B-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.014366] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3B7-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:57.109318] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5922-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.141428] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF103-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.153835] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF0A4-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.161374] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EE-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.163905] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EE-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:57.163925] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EE-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.190547] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D5B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.193897] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88BA-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.196377] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88BA-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:57.196394] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88BA-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.205078] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA10-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.211697] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=18] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:57.222226] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB81-0-0] [lt=41] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.222728] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=41] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:57.223655] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [727959][T1018_TimerWK3_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=13] mview_mds_op(tx_count=0) [2025-08-15 10:16:57.223667] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [727959][T1018_TimerWK3_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=9] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=839) [2025-08-15 10:16:57.224781] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB81-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:57.224800] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB81-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.277526] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619486][T1012_TimerWK1_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=27] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:57.281178] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BC-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.284193] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BC-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.286121] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BC-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.286304] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BC-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.337261] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=42] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:16:57.340987] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B390C-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:57.343497] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B390C-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:57.351936] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:57.353209] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224217351860, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:16:57.353234] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224222353232, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:16:57.443469] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:57.444736] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224217443396, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:16:57.444761] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224222444759, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:16:57.446542] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68BA-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.462991] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E67-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224217462404000, start_id=1755224217462404000, end_id=1755224217462404001) [2025-08-15 10:16:57.463697] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E67-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224217462404000, v:0}, tenant_id=1017) [2025-08-15 10:16:57.464614] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:57.464627] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:57.464632] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:57.464715] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E92E-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:57.466977] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD52A3-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:57.467246] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E92E-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:57.469451] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD52A3-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:57.483190] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E475-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}) [2025-08-15 10:16:57.483237] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E475-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=53, ddl_arg=NULL) [2025-08-15 10:16:57.506241] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF101-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.506302] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E770-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.536458] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224215603609002, replayable_scn:1755224215603609002, readable_scn:1755224215603609002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224217533867, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224215603609002, replayable_scn:1755224215603609002, readable_scn:1755224215603609002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224217533867, ora_rowscn_=1755224216930527000, content_changed=true) [2025-08-15 10:16:57.536639] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:57.536652] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:57.536658] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:57.537098] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:57.537110] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:57.537116] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:57.538384] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:57.542614] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:57.542624] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:57.542632] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:57.543935] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:57.548656] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:57.548666] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:57.548674] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:57.549989] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:57.554100] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:57.554119] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:57.554127] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:57.556973] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D9-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:57.558177] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5342-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:57.560573] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86D9-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:57.560664] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5342-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:57.597643] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E70-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224217597010000, start_id=1755224217597010000, end_id=1755224217597010001) [2025-08-15 10:16:57.598296] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA672-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224217598064000, start_id=1755224217598064000, end_id=1755224217598064001) [2025-08-15 10:16:57.598320] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E70-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224217597010000, v:0}, tenant_id=1015) [2025-08-15 10:16:57.598487] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227817597277}) [2025-08-15 10:16:57.598505] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:57.598515] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:16:57.598951] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA672-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224217598064000, v:0}, tenant_id=1005) [2025-08-15 10:16:57.604527] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227817603335}) [2025-08-15 10:16:57.604544] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:57.604552] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:16:57.605761] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227817604566}) [2025-08-15 10:16:57.605776] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:16:57.605788] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:16:57.626232] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE8-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:57.628719] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE8-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:57.666990] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62EC-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:57.669502] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62EC-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:57.695508] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68BD-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.702654] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB3-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.703855] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A5B-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.705224] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB3-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:57.705244] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB3-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.706412] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A5B-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:57.706430] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A5B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.706688] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDCF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.710813] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:57.710827] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:57.710832] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:57.710849] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_time_us=194, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:16:57.755702] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDDE-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.755840] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF0A8-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.816850] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:57.818147] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224217816774, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:16:57.818175] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224222818172, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:16:57.819367] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224217817994, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:16:57.819389] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224222819387, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:16:57.826670] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36D0-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224217826408000, start_id=1755224217826408000, end_id=1755224217826408001) [2025-08-15 10:16:57.827332] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36D0-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224217826408000, v:0}, tenant_id=1011) [2025-08-15 10:16:57.853370] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D5E-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.857816] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=6] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:57.857850] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=24] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:57.857868] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=13] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:57.859325] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [1487606][T1004_TimerWK3_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=9] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:57.859354] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [1487606][T1004_TimerWK3_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=20] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:57.859368] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [1487606][T1004_TimerWK3_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:57.860548] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=13] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:57.860581] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [475910][T1008_TimerWK2_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=11] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:57.869025] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.869081] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.869403] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772E-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.869601] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C63-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.872241] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140D-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.872264] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789C-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224217437168003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224217437168003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224217437168003, v:0}}], last_dump_ts_=1755224212841455, force_dump=false) [2025-08-15 10:16:57.872289] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.872348] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772E-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.872482] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C63-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.874199] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.874279] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.874288] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789C-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.874421] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C63-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.874435] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789C-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.874439] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772E-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.874444] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC095-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224217437168003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224217437168003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224217437168003, v:0}}], last_dump_ts_=1755224212844960, force_dump=false) [2025-08-15 10:16:57.874474] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC095-0-0] [lt=23] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.874517] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C63-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.874536] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772E-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.877449] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC095-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.879382] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC095-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.879551] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC095-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.896956] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8896-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.897685] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A75-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.899524] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8896-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:57.899540] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8896-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.900264] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A75-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:57.900281] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A75-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:57.907527] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC662-0-0] [lt=13] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.934426] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.937628] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768C-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.939543] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.939647] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:57.945724] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB8-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224217945313000, start_id=1755224217945313000, end_id=1755224217945313001) [2025-08-15 10:16:57.946371] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB8-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224217945313000, v:0}, tenant_id=1013) [2025-08-15 10:16:57.954327] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224217550739, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224215765381000, replayable_scn:1755224215765381000, readable_scn:1755224215765381000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:16:57.980213] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:57.981490] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224217980132, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:16:57.981516] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224222981513, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:16:57.996867] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C5929-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:57.997570] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA0C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.015775] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3B8-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:58.066227] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:58.067490] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224218066156, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:16:58.067515] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224223067513, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:16:58.109392] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5923-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.126759] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B8-0-0] [lt=22] get number(ret=0, service_type_=0, range=1, base_id=1755224218126224000, start_id=1755224218126224000, end_id=1755224218126224001) [2025-08-15 10:16:58.127590] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B8-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224218126224000, v:0}, tenant_id=1012) [2025-08-15 10:16:58.141503] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF104-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.145579] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463C0-0-0] [lt=19] get number(ret=0, service_type_=0, range=1, base_id=1755224218145155000, start_id=1755224218145155000, end_id=1755224218145155001) [2025-08-15 10:16:58.146400] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463C0-0-0] [lt=25] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224218145155000, v:0}, tenant_id=1) [2025-08-15 10:16:58.153911] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF0A5-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.166612] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EF-0-0] [lt=37] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.169125] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EF-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:58.169144] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95EF-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.190619] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D5C-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.199097] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88BB-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.201611] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88BB-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:58.201627] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88BB-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.205114] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA11-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.227608] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB82-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.230185] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB82-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:58.230203] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB82-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.287344] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BD-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.290472] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BD-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.292397] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BD-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.292498] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BD-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.294934] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA48-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}) [2025-08-15 10:16:58.294985] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA48-0-0] [lt=33] [DDL] execute ddl like stmt(ret=0, cost=55, ddl_arg=NULL) [2025-08-15 10:16:58.348986] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B390D-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:58.351529] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B390D-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:58.446616] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68BB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.464761] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:58.464774] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:58.464780] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:58.472512] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E92F-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:58.474723] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD52A4-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:58.476145] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E92F-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:58.477177] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD52A4-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:58.506315] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF102-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.506377] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E771-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.517542] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF23C-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.536670] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:58.536683] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:58.536689] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:58.537137] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:58.537149] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:58.537154] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:58.542663] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:58.542674] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:58.542682] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:58.548764] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:58.548776] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:58.548781] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:58.550751] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D586B-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83007812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83398438GB}) [2025-08-15 10:16:58.550794] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D586B-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:16:58.554109] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:58.554122] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:58.554127] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:58.566062] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86DA-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:58.566065] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5343-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:58.568539] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86DA-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:58.568572] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5343-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:58.623917] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:58.625177] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224218623839, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:16:58.625203] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224223625201, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:16:58.634024] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE9-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:58.636528] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FE9-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:58.674970] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62ED-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:58.677467] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62ED-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:58.695579] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68BE-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.706768] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDD0-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.707955] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB4-0-0] [lt=40] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.708595] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A5C-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.710564] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB4-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:58.710580] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB4-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.710885] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:58.710898] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:58.710903] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:58.711192] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A5C-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:58.711207] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A5C-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.753736] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF6-0-0] [lt=8] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1, thread_idx=0) [2025-08-15 10:16:58.755006] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-08-15 10:16:58.755775] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDDF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.755915] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF0A9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.756223] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:58.767095] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [727957][T1018_TimerWK1_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=13] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:16:58.767130] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [727957][T1018_TimerWK1_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=24] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:58.767145] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [727957][T1018_TimerWK1_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=12] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:16:58.768366] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [727959][T1018_TimerWK3_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=13] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:16:58.769976] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:58.771112] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:58.772266] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:58.772289] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:58.772305] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:58.772316] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:58.772431] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=90] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-08-15 10:16:58.772440] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:58.772453] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:58.772464] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:58.772476] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:58.772553] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:58.772568] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:58.772577] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:58.772582] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:58.772590] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3785, tenant_id=1, ls_id={id:1}) [2025-08-15 10:16:58.772597] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:58.773806] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF7-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18801, tenant_id=1, acc_dr_task=0) [2025-08-15 10:16:58.780411] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E60-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224218779586000, start_id=1755224218779586000, end_id=1755224218779586001) [2025-08-15 10:16:58.781087] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E60-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224218779586000, v:0}, tenant_id=1016) [2025-08-15 10:16:58.853419] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D5F-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.875247] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.875415] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C64-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.875441] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.875504] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772F-0-0] [lt=9] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224218438682004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224218438682004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224218438682004, v:0}}], last_dump_ts_=1755224213845243, force_dump=false) [2025-08-15 10:16:58.875532] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772F-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.878445] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140E-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.878463] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C64-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.878474] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789D-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.878754] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772F-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224218438682004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224218438682004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224218438682004, v:0}}], last_dump_ts_=1755224213848431, force_dump=false) [2025-08-15 10:16:58.878778] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772F-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.880439] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.880488] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.880517] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C64-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.880532] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC096-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.880596] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.880647] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.880651] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C64-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.880771] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772F-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.880887] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D772F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.883528] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC096-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.885380] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC096-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.885465] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC096-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.903074] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8897-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.904048] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A76-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.906870] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8897-0-0] [lt=97] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:58.906891] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8897-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.907510] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A76-0-0] [lt=89] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:58.907527] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A76-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:58.940935] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.944063] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.946092] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768D-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.946180] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:58.996947] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C592A-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:58.997644] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA0D-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.017199] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B3B9-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:59.040433] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8277-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1005, thread_idx=0) [2025-08-15 10:16:59.041706] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1005, only_for_display=false) [2025-08-15 10:16:59.042957] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:59.056809] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.057969] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.059107] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.059130] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1005, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:59.059148] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.059160] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1005, ls_id:{id:1}, replicas:[{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:59.059262] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=80] build ls locality stat map(tenant_id=1005, ls_id={id:1}) [2025-08-15 10:16:59.059270] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.059284] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.059294] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.059301] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:59.059379] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.059393] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.059403] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.059408] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.059419] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=9] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3778, tenant_id=1005, ls_id={id:1}) [2025-08-15 10:16:59.059430] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:59.060509] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18804, tenant_id=1005, acc_dr_task=0) [2025-08-15 10:16:59.061730] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1006, only_for_display=false) [2025-08-15 10:16:59.062928] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:59.076587] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.077739] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.078898] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.078924] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:16:59.078946] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.078958] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1}, replicas:[{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:59.079069] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=86] build ls locality stat map(tenant_id=1006, ls_id={id:1}) [2025-08-15 10:16:59.079077] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.079091] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.079102] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.079109] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:59.079186] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.079202] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.079211] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.079216] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.079228] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=9] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3782, tenant_id=1006, ls_id={id:1}) [2025-08-15 10:16:59.079239] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:59.080296] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:59.101125] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.102277] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.103447] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.103468] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:16:59.103483] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.103493] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1001}, replicas:[{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:59.103601] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=84] build ls locality stat map(tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:16:59.103609] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.103620] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.103628] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.103637] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:59.103713] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.103726] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.103735] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.103742] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.103747] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3776, tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:16:59.103753] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:59.104822] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:59.109470] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C5924-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.126119] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.127276] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.128424] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.128447] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:16:59.128464] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.128476] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1002}, replicas:[{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:59.128580] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=81] build ls locality stat map(tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:16:59.128588] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.128602] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.128613] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.128625] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:59.128702] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.128717] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.128727] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.128734] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.128743] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3777, tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:16:59.128754] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:59.129840] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:16:59.141575] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF105-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.150568] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.151715] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.152849] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:16:59.152873] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:16:59.152893] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=13] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.152904] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1003}, replicas:[{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:16:59.153012] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=85] build ls locality stat map(tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:16:59.153021] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.153034] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.153044] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:16:59.153056] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:16:59.153133] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.153147] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.153157] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.153163] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:16:59.153173] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3740, tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:16:59.153184] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:16:59.153987] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF0A6-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.154293] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8278-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93768, tenant_id=1006, acc_dr_task=0) [2025-08-15 10:16:59.172461] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95F0-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.175156] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95F0-0-0] [lt=96] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:16:59.175171] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95F0-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.190695] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D5D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.198337] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA653-0-0] [lt=23] get number(ret=0, service_type_=0, range=1, base_id=1755224219197346000, start_id=1755224219197346000, end_id=1755224219197346001) [2025-08-15 10:16:59.199125] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA653-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224219197346000, v:0}, tenant_id=1006) [2025-08-15 10:16:59.204349] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88BC-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.205186] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54AA12-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.206876] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88BC-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:16:59.206892] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD88BC-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.232961] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB83-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.235467] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB83-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:16:59.235484] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB83-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.263015] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2AC-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224219262387000, start_id=1755224219262387000, end_id=1755224219262387001) [2025-08-15 10:16:59.263823] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2AC-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224219262387000, v:0}, tenant_id=1008) [2025-08-15 10:16:59.293514] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BE-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.296499] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BE-0-0] [lt=15] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224218938462003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224218938462003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224218938462003, v:0}}], last_dump_ts_=1755224214265416, force_dump=false) [2025-08-15 10:16:59.296523] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BE-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.298391] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BE-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.298505] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D83BE-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.356936] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B390E-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:59.359494] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B390E-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:59.421166] INFO [RS] gc_mview (ob_mview_maintenance_task.cpp:267) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=4] mview maintenance task gc mview success(tenant_id_=1004, round_=116351, gc_mview_cost_us_=1233, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:16:59.446694] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C68BC-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.464542] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224217936883004, replayable_scn:1755224217936883004, readable_scn:1755224217936883004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224219461209, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224217936883004, replayable_scn:1755224217936883004, readable_scn:1755224217936883004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224219461209, ora_rowscn_=1755224219294988000, content_changed=true) [2025-08-15 10:16:59.464737] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:59.464750] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:59.464755] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:59.466082] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:59.481421] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E930-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:59.482486] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD52A5-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:59.483707] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98DA9-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}) [2025-08-15 10:16:59.483753] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98DA9-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=52, ddl_arg=NULL) [2025-08-15 10:16:59.483952] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E930-0-0] [lt=9] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:59.484987] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD52A5-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:59.506390] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF103-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.506446] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E772-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.536668] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:59.536681] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:59.536687] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:59.537083] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224217844259000, replayable_scn:1755224217844259000, readable_scn:1755224217844259000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224219533709, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224217844259000, replayable_scn:1755224217844259000, readable_scn:1755224217844259000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224219533709, ora_rowscn_=1755224218877476000, content_changed=true) [2025-08-15 10:16:59.537282] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:59.537298] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:59.537308] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:59.537989] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:59.543119] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224217437168004, replayable_scn:1755224217437168004, readable_scn:1755224217437168004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224219539993, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224217437168004, replayable_scn:1755224217437168004, readable_scn:1755224217437168004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224219539993, ora_rowscn_=1755224218877476000, content_changed=true) [2025-08-15 10:16:59.543300] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:59.543311] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:59.543319] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:59.548648] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224217437168003, replayable_scn:1755224217437168003, readable_scn:1755224217437168003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224219545225, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224217437168003, replayable_scn:1755224217437168003, readable_scn:1755224217437168003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224219545225, ora_rowscn_=1755224218877476000, content_changed=true) [2025-08-15 10:16:59.548820] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:59.548830] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:59.548836] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:59.553999] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224217437168003, replayable_scn:1755224217437168003, readable_scn:1755224217437168003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224219550496, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224217437168003, replayable_scn:1755224217437168003, readable_scn:1755224217437168003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224219550496, ora_rowscn_=1755224218882747000, content_changed=true) [2025-08-15 10:16:59.554178] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:59.554190] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:59.554198] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:59.555524] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:59.573762] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86DB-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:59.573872] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5344-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:59.576269] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB86DB-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:59.576398] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5344-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:59.641852] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FEA-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:59.644350] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FEA-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:59.644401] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:16:59.645652] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224219644323, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:16:59.645682] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224224645680, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:16:59.682879] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62EE-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:16:59.685378] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62EE-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:16:59.695654] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C68BF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.706844] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFDD1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.710787] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224217437168003, replayable_scn:1755224217437168003, readable_scn:1755224217437168003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224219707135, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224217437168003, replayable_scn:1755224217437168003, readable_scn:1755224217437168003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224219707135, ora_rowscn_=1755224218877476000, content_changed=true) [2025-08-15 10:16:59.710968] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:16:59.710986] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:16:59.710994] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:16:59.712353] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:16:59.713286] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB5-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.713889] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A5D-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.715806] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB5-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:16:59.715823] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDFB5-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.716469] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A5D-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:16:59.716488] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A5D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.755432] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5EC-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224219754504000, start_id=1755224219754504000, end_id=1755224219754504001) [2025-08-15 10:16:59.755844] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDE0-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.755990] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF0AA-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.756132] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5EC-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224219754504000, v:0}, tenant_id=1003) [2025-08-15 10:16:59.853494] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D60-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.881642] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140F-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.881637] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789E-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224219770265000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224219770265000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224219770265000, v:0}}], last_dump_ts_=1755224214850699, force_dump=false) [2025-08-15 10:16:59.881661] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789E-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.881664] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C65-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224219439087004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224219439087004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224219439087004, v:0}}], last_dump_ts_=1755224214851384, force_dump=false) [2025-08-15 10:16:59.881692] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C65-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.881840] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7730-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.884789] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789E-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.884947] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140F-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.885328] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7730-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.885333] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C65-0-0] [lt=16] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224219439087004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224219439087005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224219439087005, v:0}}], last_dump_ts_=1755224214854377, force_dump=false) [2025-08-15 10:16:59.885358] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C65-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.886459] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC097-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.886794] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.886877] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB789E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.886929] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.887057] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84140F-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.887350] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7730-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.887452] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D7730-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.887475] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C65-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.887554] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C65-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.889419] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC097-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.891264] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC097-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.891366] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC097-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.898282] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C9-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224219897726000, start_id=1755224219897726000, end_id=1755224219897726001) [2025-08-15 10:16:59.899074] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C9-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224219897726000, v:0}, tenant_id=1004) [2025-08-15 10:16:59.909700] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8898-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.910226] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A77-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.912367] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8898-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:16:59.912385] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8898-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.912776] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A77-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:16:59.912793] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A77-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:16:59.912950] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC663-0-0] [lt=19] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.947252] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768E-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224219607325001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224219607325002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224219607325002, v:0}}], last_dump_ts_=1755224214914953, force_dump=false) [2025-08-15 10:16:59.947277] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768E-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.950390] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768E-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224219607325001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224219607325002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224219607325002, v:0}}], last_dump_ts_=1755224214918754, force_dump=false) [2025-08-15 10:16:59.950412] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768E-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.952407] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.952511] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D768E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:16:59.997022] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C592B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:16:59.997721] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64AA0E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.080059] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99E6-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.080927] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF3E-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.082741] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99E6-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:00.082760] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99E6-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.083465] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF3E-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:00.083482] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF3E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.100699] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58AD-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.133050] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF08E-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.137642] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7827-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.137651] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841398-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.137666] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC020-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.138209] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76B9-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.138255] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BEE-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.141066] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841398-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.141140] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7827-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.141563] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BEE-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.141626] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC020-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.142833] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76B9-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.143026] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841398-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.143150] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841398-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.143215] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7827-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.143311] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7827-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.143707] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC020-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.143762] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BEE-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.143817] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC020-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.143845] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BEE-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.144954] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76B9-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.145087] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76B9-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.145486] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF02F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.146936] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227700145732}) [2025-08-15 10:15:00.146957] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=16][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:00.146965] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:15:00.152858] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227700151696}) [2025-08-15 10:15:00.152872] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:00.152883] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:15:00.154057] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227700152894}) [2025-08-15 10:15:00.154070] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:00.154082] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:15:00.163886] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2BE-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224100163056000, start_id=1755224100163056000, end_id=1755224100163056001) [2025-08-15 10:15:00.164571] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2BE-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224100163056000, v:0}, tenant_id=1007) [2025-08-15 10:15:00.182225] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CE6-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.196756] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A99B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.204273] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7617-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.207316] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7617-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.209378] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7617-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.209458] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7617-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.265160] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BA3-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87304688GB}) [2025-08-15 10:15:00.265205] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BA3-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:15:00.271519] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A00-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.271523] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8821-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.274098] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A00-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:00.274118] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A00-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.274188] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8821-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:00.274206] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8821-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.313158] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [17739][T1_TimerWK2_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=14] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:00.313188] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [17739][T1_TimerWK2_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=21] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:00.313203] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [17739][T1_TimerWK2_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:00.313226] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=13] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:00.407801] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3898-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:00.411546] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3898-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:00.438060] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6845-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.458082] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:00.458094] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:00.458100] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:00.497778] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E6FB-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.497895] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF08C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.501023] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF201-0-0] [lt=10] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.514672] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1258-0-0] [lt=6] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80859375GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.8125GB}) [2025-08-15 10:15:00.514714] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1258-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:00.530223] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:00.530241] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:00.530251] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:00.530792] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:00.530805] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:00.530813] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:00.533548] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD522F-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:00.535697] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8BA-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:00.536107] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD522F-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:00.536641] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224098770737002, replayable_scn:1755224098770737002, readable_scn:1755224098770737002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224100533534, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224098770737002, replayable_scn:1755224098770737002, readable_scn:1755224098770737002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224100533534, ora_rowscn_=1755224100140048000, content_changed=true) [2025-08-15 10:15:00.536821] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:00.536840] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:00.536848] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:00.538278] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8BA-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:00.542254] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:00.542266] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:00.542271] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:00.547561] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:00.547571] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:00.547579] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:00.552661] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957A-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.555201] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957A-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:00.555219] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.563813] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8348-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.566909] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8348-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.568634] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8846-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.568814] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8348-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.568897] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8348-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:00.571218] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8846-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:00.571236] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8846-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.596842] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB0D-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.599406] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB0D-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:00.599425] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB0D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:00.604693] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52CE-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:00.605169] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8665-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:00.607216] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52CE-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:00.607435] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:00.607683] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8665-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:00.608664] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224100607352, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:15:00.608690] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224105608688, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:15:00.640719] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [438022][T1006_TimerWK1_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=7] mview_mds_op(tx_count=0) [2025-08-15 10:15:00.640734] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [438022][T1006_TimerWK1_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=11] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=834) [2025-08-15 10:15:00.641974] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438022][T1006_TimerWK1_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=8] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:00.686650] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6848-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.695278] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F74-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:00.697811] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F74-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:00.697950] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD5A-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.705148] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:00.705163] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:00.705168] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:00.733292] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6278-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:00.735810] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6278-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:00.747316] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD69-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.747316] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF033-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.845063] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CE9-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.849928] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B343-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:00.892305] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224100544058, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224098769684004, replayable_scn:1755224098769684004, readable_scn:1755224098769684004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:00.909663] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:00.910911] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224100909590, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:15:00.910937] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224105910935, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:15:00.924840] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CD-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1015, thread_idx=0) [2025-08-15 10:15:00.926111] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1015, only_for_display=false) [2025-08-15 10:15:00.927336] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:00.935458] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:00.936689] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224100935381, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:15:00.936712] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224105936710, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:15:00.936725] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224105936722, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:00.941149] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:00.942342] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:00.943505] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:00.943528] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1015, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:00.943544] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.943555] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1015, ls_id:{id:1}, replicas:[{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:00.943659] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=82] build ls locality stat map(tenant_id=1015, ls_id={id:1}) [2025-08-15 10:15:00.943668] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:00.943681] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:00.943691] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:00.943701] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:00.943780] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.943794] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.943803] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.943811] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.943821] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3869, tenant_id=1015, ls_id={id:1}) [2025-08-15 10:15:00.943832] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:00.944899] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18788, tenant_id=1015, acc_dr_task=0) [2025-08-15 10:15:00.946197] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1016, only_for_display=false) [2025-08-15 10:15:00.947425] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=23] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:00.961271] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:00.962438] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:00.963606] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:00.963629] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:00.963646] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.963657] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1}, replicas:[{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:00.963760] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=81] build ls locality stat map(tenant_id=1016, ls_id={id:1}) [2025-08-15 10:15:00.963769] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:00.963782] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:00.963791] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:00.963803] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:00.963881] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.963895] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.963904] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.963910] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.963918] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3819, tenant_id=1016, ls_id={id:1}) [2025-08-15 10:15:00.963929] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:00.965031] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:00.986022] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:00.987169] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:00.988281] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58B4-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.988319] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:00.988343] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:00.988360] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.988371] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1001}, replicas:[{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:00.988478] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=84] build ls locality stat map(tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:15:00.988486] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:00.988501] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:00.988506] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:00.988514] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:00.988591] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.988600] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.988607] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.988614] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:00.988621] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3752, tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:15:00.988628] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:00.988953] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A997-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:00.989723] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:01.010838] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:01.012096] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:01.013315] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:01.013341] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:01.013356] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:01.013370] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1002}, replicas:[{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:01.013484] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=89] build ls locality stat map(tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:15:01.013493] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:01.013507] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:01.013518] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:01.013530] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:01.013610] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:01.013625] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:01.013635] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:01.013640] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:01.013649] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=4041, tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:15:01.013663] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=11] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:01.015050] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:01.036806] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:01.038011] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:01.039191] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:01.039214] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:01.039234] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:01.039249] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1003}, replicas:[{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:01.039358] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=86] build ls locality stat map(tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:15:01.039366] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:01.039379] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:01.039386] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:01.039397] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:01.039474] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:01.039489] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:01.039499] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:01.039504] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:01.039513] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3894, tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:15:01.039523] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:01.040750] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=95837, tenant_id=1016, acc_dr_task=0) [2025-08-15 10:15:01.086254] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99E7-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.086245] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF3F-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.088896] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF3F-0-0] [lt=73] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:01.088913] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF3F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.089523] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99E7-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:01.089544] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99E7-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.100774] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58AE-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.133119] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF08F-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.144164] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841399-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224100772425004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224100772425004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224100772425004, v:0}}], last_dump_ts_=1755224096112375, force_dump=false) [2025-08-15 10:15:01.144192] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841399-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.144378] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7828-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.144764] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC021-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.144835] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BEF-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.145560] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF030-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.146021] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BA-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.147614] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841399-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224100772425004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224100772425004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224100772425004, v:0}}], last_dump_ts_=1755224096115560, force_dump=false) [2025-08-15 10:15:01.147638] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841399-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.147675] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7828-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.148118] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC021-0-0] [lt=21] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224100772425003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224100772425004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224100772425004, v:0}}], last_dump_ts_=1755224096115675, force_dump=false) [2025-08-15 10:15:01.148140] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC021-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.148297] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BEF-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.149721] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7828-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.149786] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841399-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.149815] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7828-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.149815] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BA-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.149874] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D841399-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.150326] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC021-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.150367] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BEF-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.150451] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC021-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.150488] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BEF-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.151908] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BA-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.152039] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BA-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.167425] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:01.168665] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224101167348, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:15:01.168690] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224106168688, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:15:01.175729] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:01.176929] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224101175645, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:15:01.176958] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224106176956, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:01.182301] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CE7-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.196832] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A99C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.210540] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7618-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.213594] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7618-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.215605] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7618-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.215716] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7618-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.276755] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A01-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.276937] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8822-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.279307] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A01-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:01.279326] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A01-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.279560] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8822-0-0] [lt=68] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:01.279581] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8822-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.336916] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [727235][T1015_BackupDat][T1015][YB420ACB0122-000633A34EF59993-0-0] [lt=15] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:01.373433] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E4B-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224101373019000, start_id=1755224101373019000, end_id=1755224101373019001) [2025-08-15 10:15:01.374312] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E4B-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224101373019000, v:0}, tenant_id=1018) [2025-08-15 10:15:01.377972] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [727234][T1015_BACKUP_SC][T1015][YB420ACB0122-000633A34EE2C268-0-0] [lt=13] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:15:01.379197] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [727234][T1015_BACKUP_SC][T1015][YB420ACB0122-000633A34EE2C268-0-0] [lt=18] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:01.379257] INFO [RS] get_need_reload_task (ob_backup_data_scheduler.cpp:99) [727234][T1015_BACKUP_SC][T1015][YB420ACB0122-000633A34EE2C268-0-0] [lt=14] [DATA_BACKUP]no job need to reload(tenant_id=1015) [2025-08-15 10:15:01.380434] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [727235][T1015_BackupDat][T1015][YB420ACB0122-000633A34EF59994-0-0] [lt=15] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:01.381616] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [727235][T1015_BackupDat][T1015][YB420ACB0122-000633A34EF59995-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:01.416924] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3899-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:01.419473] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B3899-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:01.438138] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6846-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.438926] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438128][T1005_DRService][T1005][YB420ACB0122-0006338209C70E44-0-0] [lt=15] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1005, thread_idx=1) [2025-08-15 10:15:01.443250] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [438128][T1005_DRService][T1005][YB420ACB0122-0006338209C70E45-0-0] [lt=18] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2552, tenant_id=1005, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:01.445674] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [438128][T1005_DRService][T1005][YB420ACB0122-0006338209C70E45-0-0] [lt=16] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2402, tenant_id=1006, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:01.455208] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A06AD-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}) [2025-08-15 10:15:01.455251] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A06AD-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:15:01.458127] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:01.458139] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:01.458144] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:01.497849] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E6FC-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.497965] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF08D-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.530027] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224099949745003, replayable_scn:1755224099949745003, readable_scn:1755224099949745003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224101526576, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224099949745003, replayable_scn:1755224099949745003, readable_scn:1755224099949745003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224101526576, ora_rowscn_=1755224101211976000, content_changed=true) [2025-08-15 10:15:01.530235] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:01.530247] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:01.530254] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:01.530710] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224099848754001, replayable_scn:1755224099848754001, readable_scn:1755224099848754001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224101527629, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224099848754001, replayable_scn:1755224099848754001, readable_scn:1755224099848754001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224101527629, ora_rowscn_=1755224101145651000, content_changed=true) [2025-08-15 10:15:01.530889] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:01.530901] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:01.530906] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:01.536875] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:01.536887] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:01.536895] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:01.541344] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5230-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:01.542344] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:01.542357] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:01.542363] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:01.543551] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8BB-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:01.543912] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5230-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:01.546083] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8BB-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:01.548812] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:01.548828] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:01.548838] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:01.557934] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957B-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.560500] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957B-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:01.560516] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957B-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.569909] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8349-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.572929] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8349-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.574056] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8847-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.574910] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8349-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.575056] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8349-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:01.576697] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8847-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:01.576716] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8847-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.602205] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB0E-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.604783] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB0E-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:01.604800] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB0E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:01.612483] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52CF-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:01.612989] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8666-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:01.615520] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8666-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:01.615595] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52CF-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:01.619794] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC628-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.686731] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6849-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.698024] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD5B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.703162] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F75-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:01.704379] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:01.704394] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:01.704406] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:01.704414] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_time_us=177, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:01.705661] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F75-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:01.741144] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6279-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:01.743643] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=9] mview_mds_op(tx_count=0) [2025-08-15 10:15:01.743649] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6279-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:01.743655] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=9] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=864) [2025-08-15 10:15:01.744878] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475908][T1008_TimerWK0_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=16] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:01.747389] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD6A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.747393] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF034-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.766123] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:01.768539] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7F9B-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224101767673000, start_id=1755224101767673000, end_id=1755224101767673001) [2025-08-15 10:15:01.769369] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7F9B-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224101767673000, v:0}, tenant_id=1014) [2025-08-15 10:15:01.816529] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:01.817778] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224101816450, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:15:01.817803] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224106817801, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:15:01.845141] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CEA-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.851357] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B344-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:01.900754] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [476032][T1007_BackupDat][T1007][YB420ACB0122-0006338212777C1B-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:01.913650] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [476031][T1007_BACKUP_SC][T1007][YB420ACB0122-000633821262C242-0-0] [lt=12] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:15:01.988361] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58B5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:01.989031] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A998-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.007226] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E4-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1013, thread_idx=0) [2025-08-15 10:15:02.008568] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1013, only_for_display=false) [2025-08-15 10:15:02.009830] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:02.024457] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.025699] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.026867] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.026889] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1013, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:02.026904] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.026912] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=5] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1013, ls_id:{id:1}, replicas:[{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:02.027039] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=99] build ls locality stat map(tenant_id=1013, ls_id={id:1}) [2025-08-15 10:15:02.027047] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.027059] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.027067] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.027077] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:02.027152] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.027165] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.027174] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.027181] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.027188] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3894, tenant_id=1013, ls_id={id:1}) [2025-08-15 10:15:02.027196] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:02.028269] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19701, tenant_id=1013, acc_dr_task=0) [2025-08-15 10:15:02.029506] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1014, only_for_display=false) [2025-08-15 10:15:02.030718] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:02.044560] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.045732] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.046960] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.046982] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:02.046998] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.047008] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1}, replicas:[{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:02.047110] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=80] build ls locality stat map(tenant_id=1014, ls_id={id:1}) [2025-08-15 10:15:02.047118] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.047130] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.047138] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.047149] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:02.047230] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.047243] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.047250] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.047257] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.047264] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3862, tenant_id=1014, ls_id={id:1}) [2025-08-15 10:15:02.047270] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:02.048384] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:02.069498] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.070657] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.071816] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.071839] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:02.071856] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.071867] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1001}, replicas:[{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:02.071984] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=92] build ls locality stat map(tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:15:02.071993] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.072006] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.072012] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.072022] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:02.072099] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.072113] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.072123] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.072134] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.072140] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3823, tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:15:02.072150] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:02.073279] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:02.079543] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727134][T1016_TimerWK2_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:02.091705] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF40-0-0] [lt=37] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.092229] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99E8-0-0] [lt=47] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.094299] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF40-0-0] [lt=72] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:02.094318] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF40-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.094384] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.094758] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99E8-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:02.094777] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99E8-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.095542] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.096695] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.096717] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:02.096735] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.096746] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1002}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:02.096848] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=80] build ls locality stat map(tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:15:02.096857] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.096870] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.096880] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.096892] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:02.096973] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.096987] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.096996] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.097001] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.097010] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3782, tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:15:02.097019] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:02.098101] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:02.100854] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58AF-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.119115] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.120308] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.121461] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:02.121485] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:02.121502] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.121513] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1003}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:02.121622] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=86] build ls locality stat map(tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:15:02.121631] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.121644] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.121655] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:02.121666] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:02.121744] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.121758] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.121768] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.121773] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:02.121782] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3849, tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:15:02.121790] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:02.122901] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94614, tenant_id=1014, acc_dr_task=0) [2025-08-15 10:15:02.133196] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF090-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.145631] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF031-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.150857] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7829-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.150992] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139A-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.151425] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC022-0-0] [lt=10] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224101773998002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224101773998003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224101773998003, v:0}}], last_dump_ts_=1755224097118796, force_dump=false) [2025-08-15 10:15:02.151442] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF0-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.151455] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC022-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.152936] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BB-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.154001] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7829-0-0] [lt=19] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224101773998002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224101773998002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224101773998002, v:0}}], last_dump_ts_=1755224097121936, force_dump=false) [2025-08-15 10:15:02.154026] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7829-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.154031] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139A-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.154464] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF0-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.154529] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC022-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.155969] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BB-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.155970] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7829-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.156050] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.156103] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7829-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.156155] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.156463] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC022-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.156465] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF0-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.156541] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF0-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.156547] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC022-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.157923] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BB-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.158042] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BB-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.182371] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CE8-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.192036] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727959][T1018_TimerWK3_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=29] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:02.196901] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A99D-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.216664] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7619-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.219598] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7619-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.221481] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7619-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.221573] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7619-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.246310] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619486][T1012_TimerWK1_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:15:02.265627] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BA7-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87304688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}) [2025-08-15 10:15:02.265666] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BA7-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:15:02.282310] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8823-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.282661] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A02-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.284889] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8823-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:02.284907] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8823-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.285204] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A02-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:02.285221] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A02-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.306298] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487604][T1004_TimerWK1_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=5] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:02.321120] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:02.322354] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224102321039, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:15:02.322378] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224107322377, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:15:02.338383] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC4-0-0] [lt=8] start do ObArchiveSchedulerService round(round=195065) [2025-08-15 10:15:02.340782] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC4-0-0] [lt=12][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:15:02.340798] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC4-0-0] [lt=12][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1 AND dest_no = 0 AND name = 'state', exec_tenant_id=1) [2025-08-15 10:15:02.340845] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC4-0-0] [lt=7][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:02.340859] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC4-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:15:02.340869] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC4-0-0] [lt=8][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:02.407487] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:02.408723] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224102407457, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:15:02.408745] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224107408743, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:15:02.424876] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B389A-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:02.427402] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B389A-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:02.437092] INFO [RS] prepare (ob_mview_maintenance_task.cpp:176) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=17] mview maintenance task prepare success(tenant_id_=1008, round_=105550, prepare_cost_us_=0) [2025-08-15 10:15:02.438212] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6847-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.458084] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224100271454002, replayable_scn:1755224100271454002, readable_scn:1755224100271454002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224102454596, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224100271454002, replayable_scn:1755224100271454002, readable_scn:1755224100271454002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224102454596, ora_rowscn_=1755224101571847000, content_changed=true) [2025-08-15 10:15:02.458264] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:02.458279] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:02.458285] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:02.497923] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E6FD-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.498042] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF08E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.501306] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF202-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.515224] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D125A-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.8125GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81640625GB}) [2025-08-15 10:15:02.515265] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D125A-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:02.530188] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:02.530204] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:02.530209] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:02.530903] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:02.530913] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:02.530921] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:02.532241] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:02.536338] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:02.536352] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:02.536357] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:02.542260] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224100772425004, replayable_scn:1755224100772425004, readable_scn:1755224100772425004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224102538711, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224100772425004, replayable_scn:1755224100772425004, readable_scn:1755224100772425004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224102538711, ora_rowscn_=1755224102152788000, content_changed=true) [2025-08-15 10:15:02.542438] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:02.542448] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:02.542453] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:02.543788] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:02.547526] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224100772425004, replayable_scn:1755224100772425004, readable_scn:1755224100772425004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224102544919, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224100772425004, replayable_scn:1755224100772425004, readable_scn:1755224100772425004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224102544919, ora_rowscn_=1755224102152788000, content_changed=true) [2025-08-15 10:15:02.547698] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:02.547708] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:02.547716] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:02.549286] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5231-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:02.550477] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5231-0-0] [lt=13] log restore source is empty, just skip(ret=-4018, user_tenant_id=1018) [2025-08-15 10:15:02.551388] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8BC-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:02.552949] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5231-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:02.553889] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8BC-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:02.563143] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957C-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.565678] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957C-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:02.565694] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957C-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.576035] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.578699] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:340) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834A-0-0] [lt=14] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1014, ls_id:{id:1}, sync_scn:{val:1755224102452487002, v:0}, readable_scn:{val:1755224102452487002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-08-15 10:15:02.578994] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834A-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.579491] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8848-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.580857] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.580949] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:02.580989] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:459) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834A-0-0] [lt=6] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1755224100271454002, v:0}) [2025-08-15 10:15:02.582090] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8848-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:02.582109] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8848-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.607553] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB0F-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.610096] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB0F-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:02.610113] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB0F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:02.620871] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8667-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:02.620916] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D0-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:02.623381] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D0-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:02.623407] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8667-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:02.686808] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C684A-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.698101] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD5C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.704273] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224100772425004, replayable_scn:1755224100772425004, readable_scn:1755224100772425004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224102701651, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224100772425004, replayable_scn:1755224100772425004, readable_scn:1755224100772425004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224102701651, ora_rowscn_=1755224102152788000, content_changed=true) [2025-08-15 10:15:02.704521] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=30] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:02.704541] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:02.704550] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:02.710981] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F76-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:02.714670] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F76-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:02.747460] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD6B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.747471] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF035-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.749005] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB627A-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:02.751544] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB627A-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:02.772348] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [727133][T1016_TimerWK1_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=13] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:02.772383] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [727133][T1016_TimerWK1_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=23] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:02.772399] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [727133][T1016_TimerWK1_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:02.773625] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=14] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:02.785320] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:02.786575] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224102785244, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:15:02.786599] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224107786597, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:15:02.787791] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224102786517, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:15:02.787812] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224107787811, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:15:02.845217] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CEB-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.852789] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B345-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:02.949280] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:02.950540] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224102949199, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:15:02.950562] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224107950561, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:15:02.988443] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58B6-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:02.989104] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A999-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.035036] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:03.036268] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224103034959, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:15:03.036292] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224108036290, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:15:03.097015] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF41-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.097478] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99E9-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.099547] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF41-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:03.099566] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF41-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.100009] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99E9-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:03.100027] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99E9-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.100931] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58B0-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.133270] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF091-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.145706] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF032-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.157132] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.157136] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.157548] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF1-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.157557] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC023-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.158543] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227703157332}) [2025-08-15 10:15:03.158561] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:03.158573] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:15:03.158981] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BC-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224102775173003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224102775173004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224102775173004, v:0}}], last_dump_ts_=1755224098125147, force_dump=false) [2025-08-15 10:15:03.159006] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BC-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.160196] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139B-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.160338] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782A-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.160555] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC023-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.160850] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF1-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.161925] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BC-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224102775173003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224102775173003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224102775173004, v:0}}], last_dump_ts_=1755224098128987, force_dump=false) [2025-08-15 10:15:03.161959] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BC-0-0] [lt=26] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.162194] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.162284] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.162293] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.162360] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.162612] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC023-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.162706] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC023-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.162846] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF1-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.162918] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF1-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.163855] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BC-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.163966] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BC-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.164719] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227703163515}) [2025-08-15 10:15:03.164736] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:03.164748] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:15:03.165951] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227703164760}) [2025-08-15 10:15:03.165966] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:03.165976] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:15:03.182443] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CE9-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.196976] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A99E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.222561] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.226431] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761A-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.228930] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.229078] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761A-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.287648] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8824-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.287873] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A03-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.289425] INFO [RS.LB] idle (ob_root_balancer.cpp:87) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147D-0-0] [lt=31] root balance idle(idle_time=10000000) [2025-08-15 10:15:03.289444] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:70) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147D-0-0] [lt=14] start check unit migrate finish [2025-08-15 10:15:03.289457] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147D-0-0] [lt=6] try check migrate unit finish by tenant(tenant_id=1) [2025-08-15 10:15:03.290233] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8824-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:03.290250] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8824-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.290426] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A03-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:03.290442] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A03-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.298790] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147D-0-0] [lt=7] try check migrate unit finish by tenant(tenant_id=1004) [2025-08-15 10:15:03.324427] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147D-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1006) [2025-08-15 10:15:03.349988] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147D-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1008) [2025-08-15 10:15:03.375472] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147D-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1012) [2025-08-15 10:15:03.400894] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147D-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1014) [2025-08-15 10:15:03.426375] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147D-0-0] [lt=18] try check migrate unit finish by tenant(tenant_id=1016) [2025-08-15 10:15:03.432790] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B389B-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:03.435278] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B389B-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:03.438288] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6848-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.451762] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147D-0-0] [lt=12] try check migrate unit finish by tenant(tenant_id=1018) [2025-08-15 10:15:03.455685] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A06BE-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}) [2025-08-15 10:15:03.455728] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A06BE-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=47, ddl_arg=NULL) [2025-08-15 10:15:03.458282] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:03.458300] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:03.458306] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:03.477309] INFO [RS] check (ob_alter_locality_finish_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147D-0-0] [lt=12] start to check alter locality finish [2025-08-15 10:15:03.478459] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147E-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.479603] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8147E-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.481878] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81480-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.483032] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81480-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.485305] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81482-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.486442] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81482-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.488690] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81484-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.489825] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81484-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.492087] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81486-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.493215] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81486-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.495474] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81488-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.496609] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81488-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.498002] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E6FE-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.498120] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF08F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.498865] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148A-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.500032] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148A-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.502291] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148C-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.503421] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148C-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.504602] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148C-0-0] [lt=14] start check shrink resource pool [2025-08-15 10:15:03.504619] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148D-0-0] [lt=11] start check shrink resource pool(tenant_id=1) [2025-08-15 10:15:03.505733] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148D-0-0] [lt=5][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.506859] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148D-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.506910] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148E-0-0] [lt=12] start check shrink resource pool(tenant_id=1004) [2025-08-15 10:15:03.508040] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148E-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.509177] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148E-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.509228] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148F-0-0] [lt=12] start check shrink resource pool(tenant_id=1006) [2025-08-15 10:15:03.510339] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148F-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.511477] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8148F-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.511527] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81490-0-0] [lt=13] start check shrink resource pool(tenant_id=1008) [2025-08-15 10:15:03.512637] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81490-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.513786] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81490-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.513836] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81491-0-0] [lt=13] start check shrink resource pool(tenant_id=1012) [2025-08-15 10:15:03.514937] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81491-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.516080] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81491-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.516130] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81492-0-0] [lt=12] start check shrink resource pool(tenant_id=1014) [2025-08-15 10:15:03.517249] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81492-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.518379] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81492-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.518427] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81493-0-0] [lt=10] start check shrink resource pool(tenant_id=1016) [2025-08-15 10:15:03.519529] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81493-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.520649] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81493-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.520696] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81494-0-0] [lt=10] start check shrink resource pool(tenant_id=1018) [2025-08-15 10:15:03.521799] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81494-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.522949] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81494-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.522997] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:99) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81494-0-0] [lt=11] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.523010] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81494-0-0] [lt=8] start check primary zone [2025-08-15 10:15:03.523021] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81495-0-0] [lt=6] start check primary zone(tenant_id=1) [2025-08-15 10:15:03.524112] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81495-0-0] [lt=5][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.524163] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81496-0-0] [lt=11] start check primary zone(tenant_id=1004) [2025-08-15 10:15:03.525273] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81496-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.525321] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81497-0-0] [lt=11] start check primary zone(tenant_id=1006) [2025-08-15 10:15:03.526468] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81497-0-0] [lt=6][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.526520] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81498-0-0] [lt=14] start check primary zone(tenant_id=1008) [2025-08-15 10:15:03.527624] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81498-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.527673] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81499-0-0] [lt=12] start check primary zone(tenant_id=1012) [2025-08-15 10:15:03.528773] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81499-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.528821] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149A-0-0] [lt=12] start check primary zone(tenant_id=1014) [2025-08-15 10:15:03.529923] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149A-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.529977] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149B-0-0] [lt=15] start check primary zone(tenant_id=1016) [2025-08-15 10:15:03.530230] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:03.530254] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:03.530264] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:03.530922] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:03.530947] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:03.530957] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:03.531094] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149B-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.531140] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=10] start check primary zone(tenant_id=1018) [2025-08-15 10:15:03.532249] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:03.532300] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:166) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=13] start do unit balance [2025-08-15 10:15:03.536466] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:03.536477] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:03.536482] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:03.537755] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:03.538468] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:350) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=8] start distribute for server status change [2025-08-15 10:15:03.538567] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:401) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=12] finish distribute for server status change(ret=0) [2025-08-15 10:15:03.538579] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:200) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=9] finish do unit balance(ret=0) [2025-08-15 10:15:03.542532] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:03.542551] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:03.542560] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:03.544544] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:115) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=9] start to do tenant group unit balance [2025-08-15 10:15:03.544586] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=12] start to do tenant group unit balance(zone=zone1) [2025-08-15 10:15:03.544600] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=9] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.203.1.34:2882", disk_in_use_:3105882112, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:03.544619] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=13] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:03.544649] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=12] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:03.544666] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=9] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:03.544676] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone1) [2025-08-15 10:15:03.544687] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=7] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:03.544708] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=13] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.1.34:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:03.544750] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=34] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.203.1.34:2882", disk_in_use_:3105882112, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:03.544759] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=6] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.1.34:2882", disk_in_use_:3105882112, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:03.544773] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:03.544786] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=10] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1006, unit_config:{unit_config_id:1005, name:"config_opsone_zone1_s_2_4_ywx", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7988, unit_config:0x7f1e9aa71c38, pool:0x7f1c3beb2f18}, {unit_id:1008, tenant_id:1006, resource_pool_id:1008, unit_config:{unit_config_id:1008, name:"config_test_zone1_s_2_4_emp", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f77d8, unit_config:0x7f1e9aa71998, pool:0x7f1c3beb27a8}, {unit_id:1011, tenant_id:1008, resource_pool_id:1011, unit_config:{unit_config_id:1011, name:"config_mcp_zone1_s_4_8_tmc", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7550, unit_config:0x7f1e9aa716f8, pool:0x7f1e443fda40}, {unit_id:1016, tenant_id:1012, resource_pool_id:1016, unit_config:{unit_config_id:1017, name:"config_oms_zone1_s_4_8_gzp", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f73a0, unit_config:0x7f1e9aa71458, pool:0x7f1e443fd2d0}, {unit_id:1019, tenant_id:1014, resource_pool_id:1019, unit_config:{unit_config_id:1019, name:"config_grafana_test_zone1_s_1_2_pri", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7118, unit_config:0x7f1e9aa71298, pool:0x7f1e443fc7a8}, {unit_id:1024, tenant_id:1016, resource_pool_id:1024, unit_config:{unit_config_id:1023, name:"config_temporalio_zone1_s_2_4_ykw", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6ce0, unit_config:0x7f1e9aa70f18, pool:0x7f1e74dff2d0}, {unit_id:1026, tenant_id:1018, resource_pool_id:1026, unit_config:{unit_config_id:1027, name:"config_media_zone1_s_2_4_kmn", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6b30, unit_config:0x7f1e9aa70b98, pool:0x7f1e74dfeb60}], not_grant_units=[], available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:03.544844] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=35] start to do tenant group unit balance(zone=zone2) [2025-08-15 10:15:03.544854] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.203.4.38:2882", disk_in_use_:3089104896, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:03.544866] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=9] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:03.544882] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=9] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:03.544894] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=7] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:03.544903] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone2) [2025-08-15 10:15:03.544907] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:03.544921] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=10] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.38:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:03.544959] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=30] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.203.4.38:2882", disk_in_use_:3089104896, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:03.544964] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=5] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.38:2882", disk_in_use_:3089104896, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:03.544975] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=7] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:03.544985] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=8] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1004, unit_config:{unit_config_id:1004, name:"config_opsone_zone2_s_2_4_wag", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7b38, unit_config:0x7f1e9aa71d18, pool:0x7f1c3beb3688}, {unit_id:1009, tenant_id:1006, resource_pool_id:1009, unit_config:{unit_config_id:1009, name:"config_test_zone2_s_2_4_yey", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7700, unit_config:0x7f1e9aa718b8, pool:0x7f1c3beb23f0}, {unit_id:1010, tenant_id:1008, resource_pool_id:1010, unit_config:{unit_config_id:1010, name:"config_mcp_zone2_s_4_8_bjb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7628, unit_config:0x7f1e9aa717d8, pool:0x7f1c3beb2038}, {unit_id:1018, tenant_id:1012, resource_pool_id:1018, unit_config:{unit_config_id:1016, name:"config_oms_zone2_s_4_8_kbx", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f71f0, unit_config:0x7f1e9aa71538, pool:0x7f1e443fcb60}, {unit_id:1021, tenant_id:1014, resource_pool_id:1021, unit_config:{unit_config_id:1021, name:"config_grafana_test_zone2_s_1_2_yyc", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f6f68, unit_config:0x7f1e9aa710d8, pool:0x7f1e443fc038}, {unit_id:1023, tenant_id:1016, resource_pool_id:1023, unit_config:{unit_config_id:1022, name:"config_temporalio_zone2_s_2_4_lsv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6db8, unit_config:0x7f1e9aa70ff8, pool:0x7f1e74dff688}, {unit_id:1027, tenant_id:1018, resource_pool_id:1027, unit_config:{unit_config_id:1025, name:"config_media_zone2_s_2_4_yzd", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6a58, unit_config:0x7f1e9aa70d58, pool:0x7f1e74dfe7a8}], not_grant_units=[], available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:03.545043] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=33] start to do tenant group unit balance(zone=zone3) [2025-08-15 10:15:03.545053] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=7] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.203.4.39:2882", disk_in_use_:3024093184, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:03.545065] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:03.545082] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=9] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:03.545095] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=7] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:03.545103] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone3) [2025-08-15 10:15:03.545107] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:03.545121] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81640625GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.39:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81640625GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:03.545156] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=28] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.203.4.39:2882", disk_in_use_:3024093184, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:03.545162] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=4] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.39:2882", disk_in_use_:3024093184, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:03.545173] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:03.545179] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=5] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1005, unit_config:{unit_config_id:1006, name:"config_opsone_zone3_s_2_4_vxv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7a60, unit_config:0x7f1e9aa71b58, pool:0x7f1c3beb32d0}, {unit_id:1007, tenant_id:1006, resource_pool_id:1007, unit_config:{unit_config_id:1007, name:"config_test_zone3_s_2_4_mui", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f78b0, unit_config:0x7f1e9aa71a78, pool:0x7f1c3beb2b60}, {unit_id:1012, tenant_id:1008, resource_pool_id:1012, unit_config:{unit_config_id:1012, name:"config_mcp_zone3_s_4_8_seu", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7478, unit_config:0x7f1e9aa71618, pool:0x7f1e443fd688}, {unit_id:1017, tenant_id:1012, resource_pool_id:1017, unit_config:{unit_config_id:1018, name:"config_oms_zone3_s_4_8_nvb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f72c8, unit_config:0x7f1e9aa71378, pool:0x7f1e443fcf18}, {unit_id:1020, tenant_id:1014, resource_pool_id:1020, unit_config:{unit_config_id:1020, name:"config_grafana_test_zone3_s_1_2_xbz", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7040, unit_config:0x7f1e9aa711b8, pool:0x7f1e443fc3f0}, {unit_id:1022, tenant_id:1016, resource_pool_id:1022, unit_config:{unit_config_id:1024, name:"config_temporalio_zone3_s_2_4_rxz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6e90, unit_config:0x7f1e9aa70e38, pool:0x7f1e74dffa40}, {unit_id:1025, tenant_id:1018, resource_pool_id:1025, unit_config:{unit_config_id:1026, name:"config_media_zone3_s_2_4_ukz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6c08, unit_config:0x7f1e9aa70c78, pool:0x7f1e74dfef18}], not_grant_units=[], available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:03.547793] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:03.547803] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:03.547809] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:03.558233] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5232-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:03.559157] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8BD-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:03.560709] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5232-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:03.561692] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8BD-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:03.568333] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957D-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.570819] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957D-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:03.570836] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.581957] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.584835] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8849-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.585082] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834B-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224103275544003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224103275544003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224103275544003, v:0}}], last_dump_ts_=1755224098554692, force_dump=false) [2025-08-15 10:15:03.585108] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834B-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.587114] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.587281] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:03.587409] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8849-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:03.587427] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8849-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.593043] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:03.594291] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224103592968, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:15:03.594316] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224108594314, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:15:03.612866] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB10-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.615398] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB10-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:03.615414] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB10-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:03.624628] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC629-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.628748] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D1-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:03.628872] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8668-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:03.631477] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8668-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:03.632415] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D1-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:03.686885] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C684B-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.698178] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD5D-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.704515] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:03.704533] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:03.704543] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:03.705921] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:03.720001] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F77-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:03.722493] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F77-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:03.747531] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD6C-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.747551] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF036-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.756860] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB627B-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:03.760502] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB627B-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:03.845289] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CEC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.854180] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B346-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:03.893946] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224103543994, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224101773998003, replayable_scn:1755224101773998003, readable_scn:1755224101773998003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:03.988519] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58B7-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:03.989179] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A99A-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.101013] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58B1-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.102401] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF42-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.102717] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EA-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.104964] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF42-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:04.104983] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF42-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.105280] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EA-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:04.105296] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EA-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.133345] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF092-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.140238] INFO [RS] finish (ob_mview_maintenance_task.cpp:280) [438023][T1006_TimerWK2_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=7] mview maintenace task finish(tenant_id_=1006, round_=105667, status_=100, error_code_=0, cost_us_=20001402, prepare_cost_us_=0, gc_mview_cost_us_=1237, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:15:04.145780] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF033-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.163251] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.163350] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782B-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224103776371003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224103776371004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224103776371004, v:0}}], last_dump_ts_=1755224099131287, force_dump=false) [2025-08-15 10:15:04.163371] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782B-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.163674] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC024-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.163831] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF2-0-0] [lt=4] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224103776371003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224103776371004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224103776371004, v:0}}], last_dump_ts_=1755224099131454, force_dump=false) [2025-08-15 10:15:04.163851] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF2-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.164912] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BD-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.166435] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139C-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.166459] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782B-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.166664] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC024-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.166788] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF2-0-0] [lt=9] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224103776371003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224103776371004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224103776371004, v:0}}], last_dump_ts_=1755224099135157, force_dump=false) [2025-08-15 10:15:04.166807] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF2-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.167879] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BD-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.168430] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.168550] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.168636] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139C-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.168689] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC024-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.168747] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.168781] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC024-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.168804] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF2-0-0] [lt=3] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.168903] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF2-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.169777] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BD-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.169861] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BD-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.182521] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CEA-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.197050] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A99F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.230059] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761B-0-0] [lt=10] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224103776371003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224104183039000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224104183039000, v:0}}], last_dump_ts_=1755224099197988, force_dump=false) [2025-08-15 10:15:04.230082] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761B-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.233152] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761B-0-0] [lt=6] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224104183039000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224104183039000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224104183039000, v:0}}], last_dump_ts_=1755224099200980, force_dump=false) [2025-08-15 10:15:04.233172] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761B-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.235108] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761B-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.235202] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.266149] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B632D-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}) [2025-08-15 10:15:04.266189] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B632D-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:15:04.293026] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8825-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.293125] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A04-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.295572] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8825-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:04.295588] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8825-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.295639] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A04-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:04.295656] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A04-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.415224] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C52-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1011, thread_idx=0) [2025-08-15 10:15:04.416508] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1011, only_for_display=false) [2025-08-15 10:15:04.417798] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:04.431647] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.432803] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.433948] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.433972] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1011, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:04.433989] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.434002] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1011, ls_id:{id:1}, replicas:[{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:04.434104] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=79] build ls locality stat map(tenant_id=1011, ls_id={id:1}) [2025-08-15 10:15:04.434115] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.434129] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.434140] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.434152] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:04.434230] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.434241] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.434251] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.434262] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.434271] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3773, tenant_id=1011, ls_id={id:1}) [2025-08-15 10:15:04.434282] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:04.435351] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18844, tenant_id=1011, acc_dr_task=0) [2025-08-15 10:15:04.436555] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1012, only_for_display=false) [2025-08-15 10:15:04.437774] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:04.438360] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6849-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.440733] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B389C-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:04.443256] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B389C-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:04.451530] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.452681] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.453833] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.453855] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:04.453873] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.453884] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1}, replicas:[{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:04.453991] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=84] build ls locality stat map(tenant_id=1012, ls_id={id:1}) [2025-08-15 10:15:04.454007] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.454023] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=12] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.454031] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.454041] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:04.454118] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.454132] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.454143] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.454148] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.454157] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3771, tenant_id=1012, ls_id={id:1}) [2025-08-15 10:15:04.454168] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:04.455260] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:04.458369] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:04.458380] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:04.458387] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:04.459666] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:04.476138] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.477287] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.478488] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.478512] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:04.478530] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.478542] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1001}, replicas:[{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:04.478651] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=86] build ls locality stat map(tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:15:04.478660] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.478674] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.478685] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.478697] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:04.478775] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.478789] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.478799] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.478806] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.478815] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3826, tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:15:04.478827] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:04.479918] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:04.498076] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E6FF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.498169] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF090-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.500729] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.501599] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF203-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.501957] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.503114] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.503136] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:04.503155] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.503165] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1002}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:04.503273] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=85] build ls locality stat map(tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:15:04.503281] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.503290] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.503297] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.503306] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:04.503382] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.503394] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.503402] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.503409] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.503416] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3865, tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:15:04.503425] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:04.504585] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:04.515743] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D57E0-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81640625GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.8203125GB}) [2025-08-15 10:15:04.515782] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D57E0-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:15:04.525374] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=9] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.526521] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.527259] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:104) [17458][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=12] start checking lost replicas [2025-08-15 10:15:04.527664] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.527685] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:04.527699] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.527709] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1003}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:04.527809] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=79] build ls locality stat map(tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:15:04.527817] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.527828] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.527836] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.527845] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:04.527921] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.527933] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.527947] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=10] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.527954] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.527961] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3718, tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:15:04.527966] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:04.529067] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93701, tenant_id=1012, acc_dr_task=0) [2025-08-15 10:15:04.530134] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224102953818003, replayable_scn:1755224102953818003, readable_scn:1755224102953818003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224104526958, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224102953818003, replayable_scn:1755224102953818003, readable_scn:1755224102953818003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224104526958, ora_rowscn_=1755224104231394002, content_changed=true) [2025-08-15 10:15:04.530302] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:04.530312] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:04.530320] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:04.531401] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224102785642003, replayable_scn:1755224102785642003, readable_scn:1755224102785642003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224104528011, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224102785642003, replayable_scn:1755224102785642003, readable_scn:1755224102785642003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224104528011, ora_rowscn_=1755224104165189000, content_changed=true) [2025-08-15 10:15:04.531578] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:04.531588] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:04.531595] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:04.531791] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:04.536381] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224102775173004, replayable_scn:1755224102775173004, readable_scn:1755224102775173003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224104533278, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224102775173004, replayable_scn:1755224102775173004, readable_scn:1755224102775173003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224104533278, ora_rowscn_=1755224104166243000, content_changed=true) [2025-08-15 10:15:04.536534] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:04.536543] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:04.536551] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:04.542740] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:04.542751] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:04.542759] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:04.547773] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:04.547790] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:04.547797] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:04.549355] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:04.566039] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5233-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:04.566807] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8BE-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:04.568545] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5233-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:04.569272] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8BE-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:04.573529] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.576070] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957E-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:04.576088] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957E-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.588332] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834C-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224104277635002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224104277635002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224104277635002, v:0}}], last_dump_ts_=1755224099557725, force_dump=false) [2025-08-15 10:15:04.588358] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.590100] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884A-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.591298] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834C-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.592728] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884A-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:04.592746] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.593272] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.593377] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834C-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:04.613910] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:04.615130] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224104613837, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:15:04.615153] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224109615151, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:15:04.618189] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB11-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.620737] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB11-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:04.620756] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB11-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:04.636756] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8669-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:04.637566] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D2-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:04.639308] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8669-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:04.640065] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D2-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:04.677250] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=19] start do ObArchiveSchedulerService round(round=168691) [2025-08-15 10:15:04.678574] INFO [ARCHIVE] checkpoint_ (ob_tenant_archive_scheduler.cpp:682) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=16] schedule archive(round_info={key:{tenant_id:1008, dest_no:0}, incarnation:1, dest_id:1001, round_id:1, state:{status:"DOING"}, start_scn:{val:1752696006021581000, v:0}, checkpoint_scn:{val:1755223939543781004, v:0}, max_scn:{val:1755224043868820001, v:0}, compatible:{version:1}, base_piece_id:1, used_piece_id:30, piece_switch_interval:86400000000, frozen_input_bytes:13323961132, frozen_output_bytes:13323961132, active_input_bytes:117207655, active_output_bytes:117207655, deleted_input_bytes:0, deleted_output_bytes:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com", comment:""}) [2025-08-15 10:15:04.680048] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476034][T1007_ArchiveSv][T1008][YB420ACB0122-0006338212A554BE-0-0] [lt=21] get number(ret=0, service_type_=0, range=1, base_id=1755224104679472000, start_id=1755224104679472000, end_id=1755224104679472001) [2025-08-15 10:15:04.680069] INFO [SHARE] get_backup_scn (ob_backup_struct.cpp:2155) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=16] get tenant gts(ret=0, ret="OB_SUCCESS", tenant_id=1008, scn={val:1755224104679472000, v:0}) [2025-08-15 10:15:04.680089] INFO [SHARE] count_ (ob_archive_checkpoint.cpp:173) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=11] print count result(ret=0, counter={ls_count:4, deleted_ls_count:0, not_start_cnt:0, interrupted_cnt:0, doing_cnt:4, stopped_cnt:0, interrupted_ls_id:{id:-1}, max_scn:{val:1755224059715728003, v:0}, checkpoint_scn:{val:1755223995192401003, v:0}, max_active_piece_id:30}) [2025-08-15 10:15:04.681306] INFO [SHARE] get_backup_dest (ob_backup_connectivity.cpp:1268) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=13] success get backup dest(sql=SELECT authorization,extension FROM __all_backup_storage_info WHERE tenant_id = 1008 AND path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog' AND endpoint = 'host=cos.ap-beijing.myqcloud.com', tenant_id=1008, backup_dest={root_path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog", storage_info:{endpoint:"host=cos.ap-beijing.myqcloud.com", access_id:"access_id=AKIDiKl3A5ZXJ959EMuL3hJ1g9PTM5lFvjhT", extension:"appid=1254236265&checksum_type=md5", type:"COS", checksum_type:1, max_iops:0, max_bandwidth:0, role_arn:0x7f1668008f90, external_id:0x7f1668009090}}) [2025-08-15 10:15:04.686960] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C684C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.698251] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD5E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.704544] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:04.704555] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:04.704561] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:04.704569] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_time_us=176, return_code_array=[0, 0, 0], broadcast_times=2) [2025-08-15 10:15:04.713269] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB8E-0-0] [lt=10] [RESTORE] restore scheduler quit [2025-08-15 10:15:04.714556] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB8E-0-0] [lt=13] get targets(ret=0, jobs=[], exec_tenant_id=1013, sql=select * from __all_recover_table_job) [2025-08-15 10:15:04.715737] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB8F-0-0] [lt=14] get targets(ret=0, jobs=[], exec_tenant_id=1013, sql=select * from __all_import_table_job) [2025-08-15 10:15:04.715761] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB90-0-0] [lt=14][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224104715756, ref_timestamp=1755224044710945, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:15:04.715848] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB91-0-0] [lt=12] [RESTORE] restore scheduler start [2025-08-15 10:15:04.715859] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB92-0-0] [lt=9] [RESTORE] try process restore job [2025-08-15 10:15:04.717034] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [709630][T1013_REST_SER][T1013][YB420ACB0122-000633A33E67EB92-0-0] [lt=4] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:15:04.725120] INFO mk_parent_dir (ob_backup_io_adapter.cpp:346) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=20] found parent dir(i=130, path="cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog/piece_d1001r1p30/checkpoint") [2025-08-15 10:15:04.727789] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F78-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:04.730258] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F78-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:04.747607] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD6D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.747627] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF037-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.756405] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED0-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1007, thread_idx=0) [2025-08-15 10:15:04.757674] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1007, only_for_display=false) [2025-08-15 10:15:04.758949] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:04.765862] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB627C-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:04.768416] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB627C-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:04.772934] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.774116] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.775335] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.775357] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1007, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:04.775376] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.775385] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1007, ls_id:{id:1}, replicas:[{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:04.775488] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=80] build ls locality stat map(tenant_id=1007, ls_id={id:1}) [2025-08-15 10:15:04.775500] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.775513] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.775523] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.775535] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:04.775615] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.775631] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.775638] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.775648] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.775656] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3887, tenant_id=1007, ls_id={id:1}) [2025-08-15 10:15:04.775668] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:04.776769] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19095, tenant_id=1007, acc_dr_task=0) [2025-08-15 10:15:04.777991] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1008, only_for_display=false) [2025-08-15 10:15:04.779275] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:04.782393] INFO [STORAGE] print_access_storage_log (ob_storage.cpp:46) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=11] access storage op=write_single_file uri=cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog/piece_d1001r1p30/checkpoint/checkpoint_info.1755223995192401003.obarc size=0 Byte cost_ts=57246 us speed=0.00 MB/s [2025-08-15 10:15:04.782407] INFO [SHARE] write_checkpoint_file_ (ob_archive_checkpoint_mgr.cpp:255) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=10] succeed to write checkpoint file.(path={cur_pos:172, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog/piece_d1001r1p30/checkpoint/checkpoint_info.1755223995192401003.obarc"}) [2025-08-15 10:15:04.793190] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.794353] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.795536] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.795558] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:04.795576] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.795587] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1}, replicas:[{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:04.795691] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=81] build ls locality stat map(tenant_id=1008, ls_id={id:1}) [2025-08-15 10:15:04.795705] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.795717] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.795729] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.795741] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:04.795821] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.795837] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.795847] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.795857] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.795866] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3846, tenant_id=1008, ls_id={id:1}) [2025-08-15 10:15:04.795875] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:04.796950] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:04.817901] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.819126] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.820346] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.820367] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:04.820384] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.820395] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1001}, replicas:[{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:04.820499] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=80] build ls locality stat map(tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:15:04.820510] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.820523] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.820534] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.820546] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:04.820627] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.820642] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.820649] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.820659] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.820666] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3960, tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:15:04.820678] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:04.821774] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:04.842663] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.843819] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.844972] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.844995] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:04.845011] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.845023] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1002}, replicas:[{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:04.845129] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=83] build ls locality stat map(tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:15:04.845140] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.845154] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.845165] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.845176] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:04.845255] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.845271] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.845278] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.845287] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.845296] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3772, tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:15:04.845306] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:04.845364] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CED-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.846379] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:04.855559] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B347-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:04.867504] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.868667] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.869820] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:04.869844] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:04.869862] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.869875] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1003}, replicas:[{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:04.870003] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=100] build ls locality stat map(tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:15:04.870011] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.870022] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.870033] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:04.870043] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:04.870122] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.870137] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.870148] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.870155] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:04.870164] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3814, tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:15:04.870171] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:04.871247] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=10] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94463, tenant_id=1008, acc_dr_task=0) [2025-08-15 10:15:04.875321] WDIAG [STORAGE] print_access_storage_log_ (ob_i_storage.cpp:907) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=8][errcode=0] access object storage cost too much time: del_file (ob_storage.cpp:839), uri=cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog/piece_d1001r1p30/checkpoint/checkpoint_info.1755223939543781004.obarc, size=0 byte, start_time=1755224104782430, cost_ts=92890 us, speed=0.00 MB/s, is_slow=1 [2025-08-15 10:15:04.876586] INFO [SHARE] get_backup_dest (ob_backup_connectivity.cpp:1268) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=14] success get backup dest(sql=SELECT authorization,extension FROM __all_backup_storage_info WHERE tenant_id = 1008 AND path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog' AND endpoint = 'host=cos.ap-beijing.myqcloud.com', tenant_id=1008, backup_dest={root_path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog", storage_info:{endpoint:"host=cos.ap-beijing.myqcloud.com", access_id:"access_id=AKIDiKl3A5ZXJ959EMuL3hJ1g9PTM5lFvjhT", extension:"appid=1254236265&checksum_type=md5", type:"COS", checksum_type:1, max_iops:0, max_bandwidth:0, role_arn:0x7f1ae7210f90, external_id:0x7f1ae7211090}}) [2025-08-15 10:15:04.878028] INFO [SHARE] do_compare_and_swap_ (ob_inner_table_operator.cpp:937) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=19] compare and swap one column(key={key:{tenant_id:1008, dest_no:0}, incarnation:1, dest_id:1001, round_id:1, state:{status:"DOING"}, start_scn:{val:1752696006021581000, v:0}, checkpoint_scn:{val:1755223939543781004, v:0}, max_scn:{val:1755224043868820001, v:0}, compatible:{version:1}, base_piece_id:1, used_piece_id:30, piece_switch_interval:86400000000, frozen_input_bytes:13323961132, frozen_output_bytes:13323961132, active_input_bytes:117207655, active_output_bytes:117207655, deleted_input_bytes:0, deleted_output_bytes:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com", comment:""}, assignments="tenant_id = 1008, dest_no = 0, round_id = 1, incarnation = 1, dest_id = 1001, status = 'DOING', start_scn = 1752696006021581000, checkpoint_scn = 1755223995192401003, max_scn = 1755224059715728003, compatible = 1, base_piece_id = 1, used_piece_id = 30, piece_switch_interval = 86400000000, frozen_input_bytes = 13323961132, frozen_output_bytes = 13323961132, active_input_bytes = 117351157, active_output_bytes = 117351157, deleted_input_bytes = 0, deleted_output_bytes = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com', comment = ''", predicates="status = 'DOING' and round_id = 1 and checkpoint_scn = 1755223939543781004", affected_rows=1, sql=update __all_log_archive_progress set tenant_id = 1008, dest_no = 0, round_id = 1, incarnation = 1, dest_id = 1001, status = 'DOING', start_scn = 1752696006021581000, checkpoint_scn = 1755223995192401003, max_scn = 1755224059715728003, compatible = 1, base_piece_id = 1, used_piece_id = 30, piece_switch_interval = 86400000000, frozen_input_bytes = 13323961132, frozen_output_bytes = 13323961132, active_input_bytes = 117351157, active_output_bytes = 117351157, deleted_input_bytes = 0, deleted_output_bytes = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com', comment = '' where tenant_id = 1008 AND dest_no = 0 and status = 'DOING' and round_id = 1 and checkpoint_scn = 1755223939543781004) [2025-08-15 10:15:04.878204] WDIAG [STORAGE.TRANS] mvcc_sanity_check_ (ob_mvcc_row.cpp:937) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=26][errcode=-5024] find double insert node(ret=-5024, node=this=0x7f152f633a28 trans_version={val:4611686018427387903, v:0} scn={val:4611686018427387903, v:0} tx_id={txid:141876348} prev=(nil) next=(nil) modify_count=4294967295 acc_checksum=0 version=1755222899324351 type=0 flag=128 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:1059} write_epoch=2, prev=this=0x7f152f6337d0 trans_version={val:1755224054672328001, v:0} scn={val:1755224054672328001, v:0} tx_id={txid:141875648} prev=0x7f152f633648 next=(nil) modify_count=47 acc_checksum=3988111939 version=1755222899324351 type=1 flag=4 snapshot_barrier=4611686018427387803 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:1670} write_epoch=139735022537090, snapshot_version={val:1755224104877526001, v:0}, *this={this=0x7f152f622458 latch_=locked flag=11 first_dml=DELETE last_dml=INSERT update_since_compact=0 list_head=0x7f152f6337d0 latest_compact_node=0x7f152f6337d0 max_trans_version={val:1755224054672328001, v:0} max_trans_id=141875648 max_elr_trans_version={val:1755224054672328001, v:0} max_elr_trans_id=141875648 latest_compact_ts=1755224054673616 last_compact_cnt=47 total_trans_node_cnt=48 max_modify_scn={val:1755224054672328001, v:0} min_modify_scn={val:1755222899325101000, v:0}}) [2025-08-15 10:15:04.878241] WDIAG [STORAGE.TRANS] mvcc_write (ob_mvcc_row.cpp:999) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=29][errcode=-5024] mvcc write failed(ret=-5024, node=this=0x7f152f633a28 trans_version={val:4611686018427387903, v:0} scn={val:4611686018427387903, v:0} tx_id={txid:141876348} prev=(nil) next=(nil) modify_count=4294967295 acc_checksum=0 version=1755222899324351 type=0 flag=128 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:1059} write_epoch=2, ctx={this:0x7f1a0fdc2dd0, ls_id:{id:1}, ls:0x7f1a567ca150, branch:0, timeout:1755224134778080, tablet_id:{id:350}, table_iter:0x7f1a0fdc1308, table_version:1746780010640136, mvcc_acc_ctx:{type:3, abs_lock_timeout_ts:1755224134778080, tx_lock_timeout_us:-1, snapshot:{version:{val:1755224104877526001, v:0}, tx_id:{txid:141876348}, scn:{branch:0, seq:251}}, tx_table_guards:{tx_table_guard:{tx_table:0x7f1a567d5b10, epoch:1, mini_cache_:[0:{}, 1:{}, 2:{}, 3:{}]}, src_tx_table_guard:{tx_table:null, epoch:-1, mini_cache_:[0:{}, 1:{}, 2:{}, 3:{}]}, src_ls_handle:{ls_map_:null, ls_:null, mod_:-1}}, tx_id:{txid:141876348}, tx_desc:{this:0x7f1e79a517e0, tx_id:{txid:141876348}, state:2, addr:"10.203.1.34:2882", tenant_id:1007, session_id:3221553100, assoc_session_id:3221553100, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:{branch:0, seq:1}, op_sn:4, alloc_ts:1755224104877526, active_ts:1755224104877526, commit_ts:-1, finish_ts:-1, timeout_us:29999994, lock_timeout_us:-1, expire_ts:1755224134877520, coord_id:{id:-1}, parts:[{id:{id:1}, addr:"10.203.1.34:2882", epoch:509100063634033, first_scn:{branch:0, seq:250}, last_scn:{branch:0, seq:250}, last_touch_ts:1755224104877529}], exec_info_reap_ts:1755224104877528, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:1727264618, cluster_version:17180067073, seq_base:1755224104877278, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, conflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, modified_tables:[], last_rc_snapshot_version:{val:1755224104877526001, v:0}, ref:2}, tx_ctx:0x7f1aa3f1a750, mem_ctx:0x7f1aa3f1bcd0, tx_scn:{branch:0, seq:1059}, write_flag:{is_table_api:0, is_table_lock:0, is_mds:0, is_dml_batch_opt:0, is_insert_up:1, is_write_only_index:0, is_check_row_locked:0, is_lob_aux:0, is_skip_flush_redo:0, is_update_uk:0, is_update_pk_dop:0}, handle_start_time:-1, is_standby_read:false, lock_wait_start_ts:0, is_inited:true}, tablet_stat:{ls_id:0, tablet_id:0, query_cnt:0, merge_cnt:0, scan_logical_row_cnt:0, scan_physical_row_cnt:0, scan_micro_block_cnt:0, pushdown_micro_block_cnt:0, exist_row_total_table_cnt:0, exist_row_read_table_cnt:0, insert_row_cnt:0, update_row_cnt:0, delete_row_cnt:0}, is_read_store_ctx:false, update_full_column:false}) [2025-08-15 10:15:04.878315] WDIAG [STORAGE.TRANS] mvcc_write (ob_mvcc_engine.cpp:333) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=59][errcode=-5024] mvcc write failed(ret=-5024, arg={tx_id:{txid:141876348}, data:0x7f1a0fdbfcc0, old_row:null, modify_count:4294967295, acc_checksum:0, memstore_version:1755222899324351, seq_no:{branch:0, seq:1059}, write_epoch:2, scn:{val:4611686018427387903, v:0}, column_cnt:23}) [2025-08-15 10:15:04.878332] WDIAG [STORAGE.TRANS] on_key_duplication_retry (ob_memtable_context.cpp:314) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=13][errcode=0] primary key duplication conflict(key={"BIGINT":1008},{"BIGINT":1001},{"BIGINT":1},{"BIGINT":30}, this={ObIMvccCtx={alloc_type=0 ctx_descriptor=0 min_table_version=1746780010640088 max_table_version=1746780010640136 trans_version={val:4611686018427387903, v:0} commit_version={val:0, v:0} lock_wait_start_ts=0 replay_compact_version={val:0, v:0}} end_code=0 tx_status=0 is_readonly=false ref=1 trans_id={txid:141876348} ls_id=1 row_callback[alloc:1, free:0, unsubmit:1] redo[fill:0,sync_succ:0, sync_fail:0] main_list_len=3 pending_log_size=289 callback_list:{cnt=1 stat:[tx_end=0, rollback_to=0, fast_commit=0, remove_memtable=0, ext_info_log=0] detail:[(log_epoch,length,logged,synced,appended,removed,unlog_removed,branch_removed)|0:(1,3,0,0,3,0,0,0)|]}}) [2025-08-15 10:15:04.878358] WDIAG [STORAGE] insert_row_without_rowkey_check (ob_tablet.cpp:4643) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=20][errcode=-5024] fail to set memtable(ret=-5024) [2025-08-15 10:15:04.878371] WDIAG [STORAGE] insert_row_to_tablet (ob_ls_tablet_service.cpp:5667) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=9][errcode=-5024] failed to write table row(ret=-5024, table id=350, col_descs=[cnt:23, column_id=18 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=19 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=20 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=21 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=16 {type:"TIMESTAMP", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=17 {type:"TIMESTAMP", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=22 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=23 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=24 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=25 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=26 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=27 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=28 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=29 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=30 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=31 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=32 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=33 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=34 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=35 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=36 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=37 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=38 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0], datum_row={row_flag:{flag:"INSERT", flag_type:0}, trans_id:{txid:0}, scan_index:0, mvcc_row_flag:{first:0, uncommitted:0, shadow:0, compact:0, ghost:0, last:0, reserved:0, flag:0}, snapshot_version:0, fast_filter_skipped:false, have_uncommited_row:false, group_idx:0, count:23, datum_buffer:{capacity:23, datums:0x7f19d5347ea0, local_datums:0x7f19d5347af0}, datums:[col_id=0:{len: 8, flag: 0, null: 0, ptr: 0x7f19d53478a4, hex: F003000000000000, int: 1008}, col_id=1:{len: 8, flag: 0, null: 0, ptr: 0x7f19d53478ac, hex: E903000000000000, int: 1001}, col_id=2:{len: 8, flag: 0, null: 0, ptr: 0x7f19d53478b4, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=3:{len: 8, flag: 0, null: 0, ptr: 0x7f19d53478bc, hex: 1E00000000000000, int: 30}, col_id=4:{len: 8, flag: 0, null: 0, ptr: 0x7f19d53478c4, hex: D6BD45F55D3C0600, int: 1755224104877526}, col_id=5:{len: 8, flag: 0, null: 0, ptr: 0x7f19d53478cc, hex: D6BD45F55D3C0600, int: 1755224104877526}, col_id=6:{len: 8, flag: 0, null: 0, ptr: 0x7f19d53478d4, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=7:{len: 8, flag: 0, null: 0, ptr: 0x7f19d53478dc, hex: 0000000000000000, int: 0}, col_id=8:{len: 8, flag: 0, null: 0, ptr: 0x7f19d53478e4, hex: 0000000000000000, int: 0}, col_id=9:{len: 8, flag: 0, null: 0, ptr: 0x7f19d53478ec, hex: C8881CAC8FBA5B18, int: 1755201606021581000}, col_id=10:{len: 8, flag: 0, null: 0, ptr: 0x7f19d53478f4, hex: 6B5CAF8EECCE5B18, int: 1755223995192401003}, col_id=11:{len: 8, flag: 0, null: 0, ptr: 0x7f19d53478fc, hex: 83329394FBCE5B18, int: 1755224059715728003}, col_id=12:{len: 8, flag: 0, null: 0, ptr: 0x7f19d5347904, hex: C8886B3D24095C18, int: 1755288006021581000}, col_id=13:{len: 8, flag: 0, null: 0, ptr: 0x7f19d534790c, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=14:{len: 8, flag: 0, null: 0, ptr: 0x7f19d5347914, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=15:{len: 4, flag: 0, null: 0, ptr: 0x7f19d534791c, hex: 6E6F6E65, cstr: none}, col_id=16:{len: 8, flag: 0, null: 0, ptr: 0x7f19d5347920, hex: F5A2FE0600000000, int: 117351157}, col_id=17:{len: 8, flag: 0, null: 0, ptr: 0x7f19d5347928, hex: F5A2FE0600000000, int: 117351157}, col_id=18:{len: 6, flag: 0, null: 0, ptr: 0x7f19d5347930, hex: 414354495645, cstr: ACTIVE}, col_id=19:{len: 9, flag: 0, null: 0, ptr: 0x7f19d5347936, hex: 415641494C41424C45, cstr: AVAILABLE}, col_id=20:{len: 8, flag: 0, null: 0, ptr: 0x7f19d534793f, hex: 0000000000000000, int: 0}, col_id=21:{len: 8, flag: 0, null: 0, ptr: 0x7f19d5347947, hex: 0000000000000000, int: 0}, col_id=22:{len: 135, flag: 0, null: 0, ptr: 0x7f19d534794f, hex: 636F733A2F2F6261636B75702D6F6365616E626173652D313235343233363236352F6261636B75702D6F62636C75737465722F6F625F766563746F722F313732373236343631382F74656E616E745F696E6361726E6174696F6E5F312F313030382F636C6F673F686F73743D636F732E61702D6265696A696E672E6D7971636C, cstr: cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcl}]}) [2025-08-15 10:15:04.878457] WDIAG [STORAGE] insert_row (ob_ls_tablet_service.cpp:3241) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=56][errcode=-5024] failed to write row(ret=-5024) [2025-08-15 10:15:04.878484] INFO [STORAGE.TRANS] rollback (ob_memtable_context.cpp:851) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=7] memtable handle rollback to successfuly(from_seq_no={branch:0, seq:1060}, to_seq_no={branch:0, seq:1058}, remove_cnt=1, elapsed=0, this={ObIMvccCtx={alloc_type=0 ctx_descriptor=0 min_table_version=1746780010640088 max_table_version=1746780010640136 trans_version={val:4611686018427387903, v:0} commit_version={val:0, v:0} lock_wait_start_ts=0 replay_compact_version={val:0, v:0}} end_code=0 tx_status=0 is_readonly=false ref=0 trans_id={txid:141876348} ls_id=1 row_callback[alloc:1, free:0, unsubmit:1] redo[fill:0,sync_succ:0, sync_fail:0] main_list_len=2 pending_log_size=289 callback_list:{cnt=1 stat:[tx_end=0, rollback_to=1, fast_commit=0, remove_memtable=0, ext_info_log=0] detail:[(log_epoch,length,logged,synced,appended,removed,unlog_removed,branch_removed)|0:(1,2,0,0,3,1,1,0)|]}}) [2025-08-15 10:15:04.878682] INFO [SHARE] do_insert_or_update_row_ (ob_inner_table_operator.cpp:699) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BE-0-0] [lt=15] insert/update one row(row={key:{tenant_id:1008, dest_id:1001, round_id:1, piece_id:30}, incarnation:1, dest_no:0, file_count:0, start_scn:{val:1755201606021581000, v:0}, checkpoint_scn:{val:1755223995192401003, v:0}, max_scn:{val:1755224059715728003, v:0}, end_scn:{val:1755288006021581000, v:0}, compatible:{version:1}, input_bytes:117351157, output_bytes:117351157, status:{status:"ACTIVE"}, file_status:0, cp_file_id:0, cp_file_offset:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com"}, affected_rows=2, sql=INSERT INTO __all_log_archive_piece_files (tenant_id, dest_id, round_id, piece_id, incarnation, dest_no, file_count, start_scn, checkpoint_scn, max_scn, end_scn, compatible, input_bytes, output_bytes, status, file_status, cp_file_id, cp_file_offset, path) VALUES (1008, 1001, 1, 30, 1, 0, 0, 1755201606021581000, 1755223995192401003, 1755224059715728003, 1755288006021581000, 1, 117351157, 117351157, 'ACTIVE', 'AVAILABLE', 0, 0, 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com') ON DUPLICATE KEY UPDATE incarnation = 1, dest_no = 0, file_count = 0, start_scn = 1755201606021581000, checkpoint_scn = 1755223995192401003, max_scn = 1755224059715728003, end_scn = 1755288006021581000, compatible = 1, input_bytes = 117351157, output_bytes = 117351157, status = 'ACTIVE', file_status = 'AVAILABLE', cp_file_id = 0, cp_file_offset = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com') [2025-08-15 10:15:04.988590] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58B8-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.989256] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A99B-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:04.990596] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:05.024725] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [2619486][T1012_TimerWK1_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=13] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:05.024759] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [2619486][T1012_TimerWK1_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=22] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:05.024773] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [2619486][T1012_TimerWK1_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:05.026005] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [2619487][T1012_TimerWK2_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=16] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:05.101088] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58B2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.107712] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF43-0-0] [lt=37] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.107994] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EB-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.110248] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF43-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:05.110265] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF43-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.110531] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EB-0-0] [lt=72] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:05.110546] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EB-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.133413] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF093-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.145853] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF034-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.169642] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.169732] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC025-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.169732] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.169853] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF3-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.170835] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BE-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.172860] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782C-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.172918] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC025-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.173040] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF3-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.173113] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139D-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.173872] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BE-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.174786] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.174883] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.175045] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF3-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.175101] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC025-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.175130] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF3-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.175198] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC025-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.175705] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.175790] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.175799] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BE-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.175889] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BE-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.182592] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CEB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.197126] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9A0-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.236152] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761C-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.239224] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761C-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.241108] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.241205] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.298375] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8826-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.298379] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A05-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.300913] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A05-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:05.300950] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8826-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:05.300933] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A05-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.300968] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8826-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.438435] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C684A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.448647] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B389D-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:05.451169] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B389D-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:05.456166] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A08A0-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}) [2025-08-15 10:15:05.456226] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A08A0-0-0] [lt=37] [DDL] execute ddl like stmt(ret=0, cost=64, ddl_arg=NULL) [2025-08-15 10:15:05.458271] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224103275544003, replayable_scn:1755224103275544003, readable_scn:1755224103275544003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224105454682, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224103275544003, replayable_scn:1755224103275544003, readable_scn:1755224103275544003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224105454682, ora_rowscn_=1755224104590109001, content_changed=true) [2025-08-15 10:15:05.458456] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:05.458467] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:05.458474] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:05.498148] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E700-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.498241] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF091-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.530286] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:05.530298] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:05.530304] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:05.531033] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:05.531054] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:05.531060] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:05.536592] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:05.536602] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:05.536607] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:05.542442] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224103776371004, replayable_scn:1755224103776371004, readable_scn:1755224103776371004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224105538842, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224103776371004, replayable_scn:1755224103776371004, readable_scn:1755224103776371004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224105538842, ora_rowscn_=1755224105171771000, content_changed=true) [2025-08-15 10:15:05.542609] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:05.542622] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:05.542628] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:05.547755] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224103776371004, replayable_scn:1755224103776371004, readable_scn:1755224103776371004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224105544109, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224103776371004, replayable_scn:1755224103776371004, readable_scn:1755224103776371004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224105544109, ora_rowscn_=1755224105171771000, content_changed=true) [2025-08-15 10:15:05.547937] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:05.547962] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:05.547970] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:05.573952] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5234-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:05.574512] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8BF-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:05.576445] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5234-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:05.577015] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8BF-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:05.578812] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957F-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.581357] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957F-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:05.581377] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D957F-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.594355] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.596005] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884B-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.597280] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834D-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.598617] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884B-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:05.598636] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884B-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.599273] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.599358] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:05.608762] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:05.610023] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224105608688, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:15:05.610047] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224110610045, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:15:05.623513] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB12-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.626047] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB12-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:05.626064] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB12-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:05.629591] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC62A-0-0] [lt=18] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.645094] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB866A-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:05.645352] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D3-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:05.646562] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438021][T1006_TimerWK0_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=7] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:05.647843] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB866A-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:05.647967] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D3-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:05.687036] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C684D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.698327] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD5F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.704395] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224103776371004, replayable_scn:1755224103776371004, readable_scn:1755224103776371003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224105700838, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224103776371004, replayable_scn:1755224103776371004, readable_scn:1755224103776371003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224105700838, ora_rowscn_=1755224105171771000, content_changed=true) [2025-08-15 10:15:05.704555] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:05.704566] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:05.704573] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:05.735516] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F79-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:05.737999] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F79-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:05.747677] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD6E-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.747698] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF038-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.773811] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB627D-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:05.776339] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB627D-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:05.845439] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CEE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.856990] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B348-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:05.914840] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:05.916082] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224105910935, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:15:05.916107] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224110916105, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:15:05.936782] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:05.938020] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224105936710, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:15:05.938045] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224110938043, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:15:05.938063] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224110938060, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:05.977975] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC1291066-0-0] [lt=12] [RESTORE] restore scheduler quit [2025-08-15 10:15:05.979213] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC1291066-0-0] [lt=12] get targets(ret=0, jobs=[], exec_tenant_id=1011, sql=select * from __all_recover_table_job) [2025-08-15 10:15:05.980391] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC1291067-0-0] [lt=15] get targets(ret=0, jobs=[], exec_tenant_id=1011, sql=select * from __all_import_table_job) [2025-08-15 10:15:05.980419] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC1291068-0-0] [lt=18][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224105980414, ref_timestamp=1755224045975785, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:15:05.980507] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC1291069-0-0] [lt=12] [RESTORE] restore scheduler start [2025-08-15 10:15:05.980522] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC129106A-0-0] [lt=12] [RESTORE] try process restore job [2025-08-15 10:15:05.981678] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [2619606][T1011_REST_SER][T1011][YB420ACB0122-0006338CC129106A-0-0] [lt=4] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:15:05.988665] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58B9-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:05.989332] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A99C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.101164] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58B3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.113035] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF44-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.113255] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EC-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.115556] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF44-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:06.115575] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF44-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.115766] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EC-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:06.115782] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EC-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.133487] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF094-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.145921] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF035-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.170321] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227706169134}) [2025-08-15 10:15:06.170336] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:06.170346] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:15:06.175868] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782D-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.176068] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF4-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.176203] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC026-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.176335] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=6] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:06.176807] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139E-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224105779763003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224105779763003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224105779763003, v:0}}], last_dump_ts_=1755224101144163, force_dump=false) [2025-08-15 10:15:06.176829] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139E-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.176859] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BF-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.177096] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227706175896}) [2025-08-15 10:15:06.177112] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:06.177122] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:15:06.177571] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224106168688, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:15:06.177595] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224111177593, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:15:06.178342] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227706177132}) [2025-08-15 10:15:06.178355] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:06.178365] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:15:06.178614] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:06.179039] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782D-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.179210] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF4-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.179307] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC026-0-0] [lt=8] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224105779763002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224105779763003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224105779763003, v:0}}], last_dump_ts_=1755224101148117, force_dump=false) [2025-08-15 10:15:06.179327] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC026-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.179801] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BF-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.179823] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139E-0-0] [lt=18] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224105779763003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224105779763003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224105779763003, v:0}}], last_dump_ts_=1755224101147614, force_dump=false) [2025-08-15 10:15:06.179842] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.179858] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224106176956, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:15:06.179879] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224111179878, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:06.181069] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.181186] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.181246] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF4-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.181322] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC026-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.181362] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF4-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.181408] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC026-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.181765] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.181818] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BF-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.181862] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.181928] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76BF-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.182662] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CEC-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.197199] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9A1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.242158] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.245141] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761D-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.247021] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.247138] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.266680] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D24-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}) [2025-08-15 10:15:06.266719] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D24-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:15:06.282246] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=14] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:06.282278] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=22] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:06.282292] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:06.284755] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=8] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:06.303698] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8827-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.303697] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A06-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.306206] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A06-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:06.306225] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A06-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.306262] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8827-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:06.306279] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8827-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.370980] INFO [RS] finish (ob_mview_maintenance_task.cpp:280) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=14] mview maintenace task finish(tenant_id_=1018, round_=63407, status_=100, error_code_=0, cost_us_=20001410, prepare_cost_us_=0, gc_mview_cost_us_=1243, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:15:06.438509] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C684B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.456640] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B389E-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:06.458440] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:06.458452] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:06.458457] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:06.459189] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B389E-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:06.498223] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E701-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.498316] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF092-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.501905] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF204-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.516217] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD697D-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.8203125GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83203125GB}) [2025-08-15 10:15:06.516253] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD697D-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:15:06.530494] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:06.530507] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:06.530512] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:06.531060] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:06.531071] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:06.531076] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:06.536604] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:06.536617] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:06.536624] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:06.542677] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:06.542689] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:06.542695] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:06.547876] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:06.547889] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:06.547894] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:06.581744] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5235-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:06.582294] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C0-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:06.584045] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9580-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.584780] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C0-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:06.584832] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5235-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:06.586639] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9580-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:06.586657] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9580-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.600349] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834E-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.601365] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884C-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.603306] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.603978] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884C-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:06.603994] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884C-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.605219] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.605305] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:06.628967] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB13-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.631569] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB13-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:06.631587] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB13-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:06.653341] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB866B-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:06.653431] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D4-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:06.656027] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D4-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:06.657007] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB866B-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:06.687119] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C684E-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.698404] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD60-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.704665] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:06.704682] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:06.704689] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:06.743308] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F7A-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:06.745791] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F7A-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:06.747749] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD6F-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.747773] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF039-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.749470] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475908][T1008_TimerWK0_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:06.767471] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709536][T1014_TimerWK3_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:06.781815] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB627E-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:06.784372] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB627E-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:06.800070] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709637][T1013_DRService][T1013][YB420ACB0122-000633A341D55696-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1013, thread_idx=1) [2025-08-15 10:15:06.803666] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [709637][T1013_DRService][T1013][YB420ACB0122-000633A341D55697-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2346, tenant_id=1013, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:06.806006] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [709637][T1013_DRService][T1013][YB420ACB0122-000633A341D55697-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2321, tenant_id=1014, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:06.817871] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:06.819108] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224106817801, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:15:06.819130] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224111819129, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:15:06.845512] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CEF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.858424] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B349-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:06.895566] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224106544556, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224104778433003, replayable_scn:1755224104778433003, readable_scn:1755224104778433002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:06.954803] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [709633][T1013_BackupDat][T1013][YB420ACB0122-000633A33E8598BA-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:06.988655] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [709632][T1013_BACKUP_SC][T1013][YB420ACB0122-000633A33E72C1CC-0-0] [lt=12] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:15:06.988738] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58BA-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.989406] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A99D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:06.989869] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [709632][T1013_BACKUP_SC][T1013][YB420ACB0122-000633A33E72C1CC-0-0] [lt=13] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:06.989928] INFO [RS] get_need_reload_task (ob_backup_data_scheduler.cpp:99) [709632][T1013_BACKUP_SC][T1013][YB420ACB0122-000633A33E72C1CC-0-0] [lt=13] [DATA_BACKUP]no job need to reload(tenant_id=1013) [2025-08-15 10:15:06.991326] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [709633][T1013_BackupDat][T1013][YB420ACB0122-000633A33E8598BB-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:06.992499] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [709633][T1013_BackupDat][T1013][YB420ACB0122-000633A33E8598BC-0-0] [lt=13] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:07.084978] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=17] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:07.101237] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58B4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.118356] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF45-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.118478] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99ED-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.120986] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF45-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:07.120997] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99ED-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:07.121004] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF45-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.121013] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99ED-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.133559] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF095-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.145988] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF036-0-0] [lt=8] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.182182] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.182372] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF5-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.182383] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC027-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224106780854003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224106780854004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224106780854004, v:0}}], last_dump_ts_=1755224102151425, force_dump=false) [2025-08-15 10:15:07.182414] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC027-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.182733] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CED-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.182804] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.182845] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C0-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.185210] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782E-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224106780854004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224106780854004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224106780854004, v:0}}], last_dump_ts_=1755224102154000, force_dump=false) [2025-08-15 10:15:07.185235] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782E-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.185566] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF5-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.185627] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC027-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.185665] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139F-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.185787] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C0-0-0] [lt=25] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.187189] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782E-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.187275] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.187553] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF5-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.187649] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF5-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.187676] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC027-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.187681] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139F-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.187754] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC027-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.187766] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D84139F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.187788] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C0-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.187872] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C0-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.193341] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727957][T1018_TimerWK1_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:07.197274] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9A2-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.247631] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:07.248131] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.248573] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=10] mview_mds_op(tx_count=0) [2025-08-15 10:15:07.248586] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=9] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=851) [2025-08-15 10:15:07.251142] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.253014] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761E-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.253115] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.307629] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487606][T1004_TimerWK3_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=27] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:07.308984] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8828-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.308981] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A07-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.311501] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8828-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:07.311510] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A07-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:07.311520] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8828-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.311524] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A07-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.322451] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:07.323702] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224107322377, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:15:07.323725] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224112323723, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:15:07.405744] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E5C-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224107405427000, start_id=1755224107405427000, end_id=1755224107405427001) [2025-08-15 10:15:07.406416] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E5C-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224107405427000, v:0}, tenant_id=1017) [2025-08-15 10:15:07.408816] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:07.410051] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224107408743, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:15:07.410073] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224112410072, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:15:07.438581] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C684C-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.456634] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E38A-0-0] [lt=13] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}) [2025-08-15 10:15:07.456690] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E38A-0-0] [lt=35] [DDL] execute ddl like stmt(ret=0, cost=62, ddl_arg=NULL) [2025-08-15 10:15:07.458560] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:07.458572] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:07.458577] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:07.464764] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B389F-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:07.467856] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B389F-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:07.498293] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E702-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.498389] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF093-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.530333] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224105779763004, replayable_scn:1755224105779763004, readable_scn:1755224105779763004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224107527469, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224105779763004, replayable_scn:1755224105779763004, readable_scn:1755224105779763004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224107527469, ora_rowscn_=1755224107249909000, content_changed=true) [2025-08-15 10:15:07.530514] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:07.530524] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:07.530531] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:07.531135] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:07.531147] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:07.531153] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:07.532433] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:07.536517] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224105779763004, replayable_scn:1755224105779763004, readable_scn:1755224105779763004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224107533788, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224105779763004, replayable_scn:1755224105779763004, readable_scn:1755224105779763004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224107533788, ora_rowscn_=1755224107184684000, content_changed=true) [2025-08-15 10:15:07.536685] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:07.536695] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:07.536703] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:07.540456] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E65-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224107540098000, start_id=1755224107540098000, end_id=1755224107540098001) [2025-08-15 10:15:07.540697] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA667-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224107540098000, start_id=1755224107540098000, end_id=1755224107540098001) [2025-08-15 10:15:07.541112] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E65-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224107540098000, v:0}, tenant_id=1015) [2025-08-15 10:15:07.541399] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA667-0-0] [lt=19] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224107540098000, v:0}, tenant_id=1005) [2025-08-15 10:15:07.542716] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:07.542726] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:07.542734] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:07.544051] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:07.547943] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:07.547957] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:07.547963] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:07.589379] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9581-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.590022] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C1-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:07.590079] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5236-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:07.591890] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9581-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:07.591911] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9581-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.592575] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5236-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:07.593643] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C1-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:07.606255] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834F-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.606629] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884D-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.609170] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884D-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:07.609201] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834F-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.609186] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884D-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.611275] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.611362] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D834F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:07.634435] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC62B-0-0] [lt=13] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.634382] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB14-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.636950] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB14-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:07.636970] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB14-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:07.661440] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D5-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:07.662504] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB866C-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:07.664061] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D5-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:07.665046] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB866C-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:07.687203] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C684F-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.698485] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD61-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.704715] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:07.704726] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:07.704731] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:07.704740] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_time_us=184, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:07.747825] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD70-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.747847] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF03A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.752170] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F7B-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:07.754710] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F7B-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:07.770015] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36C5-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224107769509000, start_id=1755224107769509000, end_id=1755224107769509001) [2025-08-15 10:15:07.770704] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36C5-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224107769509000, v:0}, tenant_id=1011) [2025-08-15 10:15:07.786676] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:07.787485] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826C-0-0] [lt=17] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1005, thread_idx=0) [2025-08-15 10:15:07.788455] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224107786597, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:15:07.788486] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224112788484, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:15:07.788781] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1005, only_for_display=false) [2025-08-15 10:15:07.790001] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:07.790235] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224107787811, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:15:07.790265] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224112790263, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:15:07.790841] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB627F-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:07.793384] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB627F-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:07.803750] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.805059] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.806616] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.806644] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1005, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:07.806667] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=14] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.806682] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1005, ls_id:{id:1}, replicas:[{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:07.806848] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=130] build ls locality stat map(tenant_id=1005, ls_id={id:1}) [2025-08-15 10:15:07.806864] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.806882] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=14] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.806898] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=12] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.806915] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:07.807051] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=102] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.807071] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=10] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.807086] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=11] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.807098] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=9] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.807110] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=9] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=4508, tenant_id=1005, ls_id={id:1}) [2025-08-15 10:15:07.807125] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=12] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:07.808525] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=10] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19745, tenant_id=1005, acc_dr_task=0) [2025-08-15 10:15:07.809797] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1006, only_for_display=false) [2025-08-15 10:15:07.811005] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:07.824861] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.826005] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.827134] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.827160] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:07.827179] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.827192] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1}, replicas:[{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:07.827293] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=80] build ls locality stat map(tenant_id=1006, ls_id={id:1}) [2025-08-15 10:15:07.827301] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.827315] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.827326] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.827338] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:07.827416] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.827431] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.827443] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.827448] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.827454] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3762, tenant_id=1006, ls_id={id:1}) [2025-08-15 10:15:07.827466] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:07.828560] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:07.845583] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CF0-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.849177] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.850351] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.851514] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.851537] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:07.851553] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.851564] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1001}, replicas:[{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:07.851669] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=81] build ls locality stat map(tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:15:07.851680] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.851694] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.851705] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.851717] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:07.851795] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.851810] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.851820] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.851826] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.851835] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3848, tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:15:07.851842] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:07.852961] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:07.859790] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B34A-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:07.873713] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.874887] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.876230] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.876255] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:07.876275] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.876286] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1002}, replicas:[{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:07.876391] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=82] build ls locality stat map(tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:15:07.876399] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.876413] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.876419] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.876430] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:07.876508] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.876523] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.876534] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.876541] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.876550] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3994, tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:15:07.876561] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:07.877685] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:07.887964] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FAD-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224107887280000, start_id=1755224107887280000, end_id=1755224107887280001) [2025-08-15 10:15:07.888629] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FAD-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224107887280000, v:0}, tenant_id=1013) [2025-08-15 10:15:07.888651] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FAD-0-0] [lt=13] zone_merge_mgr is already loaded(tenant_id=1013, global_merge_info={tenant_id:1013, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401969619, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194664677655, need_update:false}}, zone_merge_infos=[{tenant_id:1013, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194664678484, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401990881, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1013, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194664680226, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401996855, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1013, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194664681976, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402000582, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401940214000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:15:07.898503] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.899645] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.900793] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:07.900816] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:07.900833] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.900844] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1003}, replicas:[{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:07.900954] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=86] build ls locality stat map(tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:15:07.900963] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.900976] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.900986] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:07.900999] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:07.901075] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.901089] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.901100] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.901107] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:07.901116] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3744, tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:15:07.901126] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:07.902180] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93636, tenant_id=1006, acc_dr_task=0) [2025-08-15 10:15:07.950633] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:07.951863] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224107950561, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:15:07.951884] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224112951883, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:15:07.988810] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58BB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:07.989481] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A99E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.036346] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:08.037623] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224108036290, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:15:08.037646] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224113037645, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:15:08.063984] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36AD-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224108063733000, start_id=1755224108063733000, end_id=1755224108063733001) [2025-08-15 10:15:08.064823] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36AD-0-0] [lt=19] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224108063733000, v:0}, tenant_id=1012) [2025-08-15 10:15:08.085575] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463B5-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224108084800000, start_id=1755224108084800000, end_id=1755224108084800001) [2025-08-15 10:15:08.086424] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463B5-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224108084800000, v:0}, tenant_id=1) [2025-08-15 10:15:08.101308] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58B5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.123773] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EE-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.123835] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF46-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.126286] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EE-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:08.126304] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EE-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.126390] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF46-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:08.126409] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF46-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.133587] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF096-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.146064] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF037-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.182808] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CEE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.188292] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.188561] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF6-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.188714] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A0-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.188725] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC028-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.188803] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C1-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224107782071004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224107782071004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224107782071004, v:0}}], last_dump_ts_=1755224103158980, force_dump=false) [2025-08-15 10:15:08.188827] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C1-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.191377] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782F-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.191605] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF6-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.191668] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A0-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.191882] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC028-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.192058] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C1-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224107782071005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224107782071005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224107782071005, v:0}}], last_dump_ts_=1755224103161924, force_dump=false) [2025-08-15 10:15:08.192081] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C1-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.193354] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.193454] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB782F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.193583] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF6-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.193657] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A0-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.193679] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF6-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.193841] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A0-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.194000] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC028-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.194120] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC028-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.194166] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C1-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.194264] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C1-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.197351] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9A3-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.218477] INFO [RS] recover_task (ob_ddl_scheduler.cpp:2771) [17347][TimerWK0_DDLScanTask][T0][Y0-0000000000000000-0-0] [lt=13] start processing ddl recovery(ddl_event_info={addr_:"10.203.1.34:2882", event_ts_:1755224108218476, sub_id_:0, trace_id_:Y0-0000000000000000-0-0, parent_trace_id_:Y0-0000000000000000-0-0}, task_records=[], primary_tenant_ids=[1004, 1006, 1008, 1012, 1014, 1016, 1018]) [2025-08-15 10:15:08.254202] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761F-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.257271] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761F-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.259232] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.259326] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D761F-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.267199] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BAC-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}) [2025-08-15 10:15:08.267244] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BAC-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=50, ddl_arg=NULL) [2025-08-15 10:15:08.314311] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8829-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.314312] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A08-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.316848] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8829-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:08.316854] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A08-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:08.316869] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A08-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.316869] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8829-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.317137] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A62F-0-0] [lt=8] start do ObArchiveSchedulerService round(round=124004) [2025-08-15 10:15:08.320636] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A62F-0-0] [lt=11][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:15:08.320654] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A62F-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1012 AND dest_no = 0 AND name = 'state', exec_tenant_id=1011) [2025-08-15 10:15:08.320697] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A62F-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1012, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:08.320718] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A62F-0-0] [lt=16][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1012, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:15:08.320730] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A62F-0-0] [lt=9][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1012, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:08.438656] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C684D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.458401] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224106279650003, replayable_scn:1755224106279650003, readable_scn:1755224106279650003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224108454968, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224106279650003, replayable_scn:1755224106279650003, readable_scn:1755224106279650003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224108454968, ora_rowscn_=1755224107607430000, content_changed=true) [2025-08-15 10:15:08.458583] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:08.458594] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:08.458601] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:08.473231] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A0-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:08.475729] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A0-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:08.498366] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E703-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.498461] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF094-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.502209] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF205-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.516713] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6982-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83203125GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}) [2025-08-15 10:15:08.516753] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6982-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:15:08.530562] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:08.530574] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:08.530578] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:08.530897] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEB-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1, thread_idx=0) [2025-08-15 10:15:08.531133] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224106780854004, replayable_scn:1755224106780854004, readable_scn:1755224106780854004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224108527512, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224106780854004, replayable_scn:1755224106780854004, readable_scn:1755224106780854004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224108527512, ora_rowscn_=1755224108189892000, content_changed=true) [2025-08-15 10:15:08.531301] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:08.531313] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:08.531323] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:08.532161] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-08-15 10:15:08.533393] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:08.536716] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:08.536728] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:08.536736] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:08.538021] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:08.542605] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224106780854004, replayable_scn:1755224106780854004, readable_scn:1755224106780854004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224108539103, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224106780854004, replayable_scn:1755224106780854004, readable_scn:1755224106780854004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224108539103, ora_rowscn_=1755224108189892000, content_changed=true) [2025-08-15 10:15:08.542780] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:08.542790] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:08.542798] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:08.547520] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.547811] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224106780854004, replayable_scn:1755224106780854004, readable_scn:1755224106780854004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224108544378, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224106780854004, replayable_scn:1755224106780854004, readable_scn:1755224106780854004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224108544378, ora_rowscn_=1755224108190946000, content_changed=true) [2025-08-15 10:15:08.547986] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:08.548001] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:08.548008] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:08.548739] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.549907] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.549929] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:08.549951] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=13] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.549961] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:08.550097] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=106] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-08-15 10:15:08.550105] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.550117] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.550126] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.550134] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:08.550210] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.550223] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.550230] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.550238] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.550244] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3913, tenant_id=1, ls_id={id:1}) [2025-08-15 10:15:08.550253] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:08.551381] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19220, tenant_id=1, acc_dr_task=0) [2025-08-15 10:15:08.594385] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:08.594613] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9582-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.595626] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224108594314, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:15:08.595652] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224113595651, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:15:08.597160] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9582-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:08.597179] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9582-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.597896] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5237-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:08.598891] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C2-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:08.600358] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5237-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:08.601384] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C2-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:08.611820] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.612345] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8350-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.614392] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884E-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:08.614410] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.615346] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8350-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224108282538003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224108282538003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224108282538002, v:0}}], last_dump_ts_=1755224103585081, force_dump=false) [2025-08-15 10:15:08.615368] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8350-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.617296] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8350-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.617376] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8350-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:08.639787] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB15-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.642387] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB15-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:08.642405] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB15-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:08.669549] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D6-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:08.670313] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB866D-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:08.672130] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:08.672802] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB866D-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:08.687281] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6850-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.698559] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD62-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.704654] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224106780854004, replayable_scn:1755224106780854004, readable_scn:1755224106780854004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224108701010, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224106780854004, replayable_scn:1755224106780854004, readable_scn:1755224106780854004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224108701010, ora_rowscn_=1755224108189892000, content_changed=true) [2025-08-15 10:15:08.704831] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:08.704844] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:08.704852] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:08.706160] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:08.723189] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E55-0-0] [lt=11] get number(ret=0, service_type_=0, range=1, base_id=1755224108723110000, start_id=1755224108723110000, end_id=1755224108723110001) [2025-08-15 10:15:08.723833] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E55-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224108723110000, v:0}, tenant_id=1016) [2025-08-15 10:15:08.747894] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD71-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.747917] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF03B-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.757718] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5E-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1003, thread_idx=0) [2025-08-15 10:15:08.758998] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1003, only_for_display=false) [2025-08-15 10:15:08.760091] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F7C-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:08.760203] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:08.762595] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F7C-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:08.773988] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.775162] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.776348] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.776372] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1003, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:08.776391] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.776404] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1003, ls_id:{id:1}, replicas:[{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:08.776510] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=83] build ls locality stat map(tenant_id=1003, ls_id={id:1}) [2025-08-15 10:15:08.776524] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.776541] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.776552] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.776564] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:08.776657] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=69] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.776668] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.776678] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.776683] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.776692] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3873, tenant_id=1003, ls_id={id:1}) [2025-08-15 10:15:08.776702] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:08.777797] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18800, tenant_id=1003, acc_dr_task=0) [2025-08-15 10:15:08.779024] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1004, only_for_display=false) [2025-08-15 10:15:08.780234] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:08.794501] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.795162] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [438124][T1005_BackupDat][T1005][YB420ACB0122-0006338207177DA2-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:08.795693] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.796857] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.796879] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:08.796897] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.796909] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:08.797022] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=89] build ls locality stat map(tenant_id=1004, ls_id={id:1}) [2025-08-15 10:15:08.797032] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.797045] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.797056] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.797068] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:08.797146] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.797160] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.797169] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.797180] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.797186] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3887, tenant_id=1004, ls_id={id:1}) [2025-08-15 10:15:08.797196] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:08.798330] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:08.798784] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6280-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:08.800374] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [438123][T1005_BACKUP_SC][T1005][YB420ACB0122-000633820702C25B-0-0] [lt=16] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:15:08.801319] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6280-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:08.819245] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.820401] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.821608] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.821632] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:08.821649] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.821660] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:08.821781] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=94] build ls locality stat map(tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:15:08.821794] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.821807] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.821821] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=10] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.821831] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:08.821908] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.821924] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.821934] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.821950] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=13] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.821955] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3886, tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:15:08.821965] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:08.823050] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:08.843968] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.845117] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.845653] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CF1-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.846313] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.846336] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:08.846353] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.846365] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1002}, replicas:[{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:08.846474] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=85] build ls locality stat map(tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:15:08.846482] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.846496] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.846507] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.846519] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:08.846597] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.846611] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.846621] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.846631] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.846636] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3826, tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:15:08.846646] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:08.847696] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:08.861245] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B34B-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:08.868597] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.869755] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.870910] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:08.870932] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:08.870954] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=15] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.870964] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1003}, replicas:[{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:08.871074] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=85] build ls locality stat map(tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:15:08.871085] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.871097] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.871105] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:08.871115] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:08.871191] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.871201] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=3] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.871208] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.871216] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:08.871223] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3764, tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:15:08.871231] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:08.872294] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94480, tenant_id=1004, acc_dr_task=0) [2025-08-15 10:15:08.898991] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619613][T1011_DRService][T1011][YB420ACB0122-0006338CC425E8FF-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1011, thread_idx=1) [2025-08-15 10:15:08.902826] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [2619613][T1011_DRService][T1011][YB420ACB0122-0006338CC425E900-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2416, tenant_id=1011, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:08.905174] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [2619613][T1011_DRService][T1011][YB420ACB0122-0006338CC425E900-0-0] [lt=16] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2326, tenant_id=1012, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:08.988885] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58BC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:08.989556] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A99F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.084128] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B7-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1017, thread_idx=0) [2025-08-15 10:15:09.085411] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1017, only_for_display=false) [2025-08-15 10:15:09.086648] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:09.100577] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.101366] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58B6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.101738] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.102902] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.102925] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1017, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:09.102949] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=15] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.102961] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1017, ls_id:{id:1}, replicas:[{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:09.103065] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=81] build ls locality stat map(tenant_id=1017, ls_id={id:1}) [2025-08-15 10:15:09.103073] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.103086] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.103097] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.103110] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:09.103188] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.103202] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.103213] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.103223] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.103229] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3826, tenant_id=1017, ls_id={id:1}) [2025-08-15 10:15:09.103240] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:09.104343] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18932, tenant_id=1017, acc_dr_task=0) [2025-08-15 10:15:09.105555] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1018, only_for_display=false) [2025-08-15 10:15:09.106764] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:09.120579] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.121760] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.122945] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.122967] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:09.122982] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.122991] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1}, replicas:[{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:09.123090] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=77] build ls locality stat map(tenant_id=1018, ls_id={id:1}) [2025-08-15 10:15:09.123097] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.123108] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.123117] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.123126] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:09.123201] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.123213] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.123221] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.123227] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.123234] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3817, tenant_id=1018, ls_id={id:1}) [2025-08-15 10:15:09.123242] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:09.124352] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:09.129030] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EF-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.129161] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF47-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.131696] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF47-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:09.131713] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF47-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.132144] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EF-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:09.132162] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99EF-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.133660] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF097-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.135645] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA648-0-0] [lt=9] get number(ret=0, service_type_=0, range=1, base_id=1755224109135451000, start_id=1755224109135451000, end_id=1755224109135451001) [2025-08-15 10:15:09.136463] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA648-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224109135451000, v:0}, tenant_id=1006) [2025-08-15 10:15:09.145359] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.146139] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF038-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.146518] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.147685] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.147709] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:09.147720] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.147726] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=5] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1001}, replicas:[{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:09.147829] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:15:09.147837] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.147850] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.147856] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=4] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.147866] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:09.147947] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.147961] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.147970] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.147975] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.147984] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3783, tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:15:09.147994] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:09.149079] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:09.170070] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.171233] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.172382] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.172405] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:09.172422] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.172433] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1002}, replicas:[{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:09.172536] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=81] build ls locality stat map(tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:15:09.172544] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.172557] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.172568] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.172575] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:09.172653] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.172670] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=10] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.172680] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.172685] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.172694] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3786, tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:15:09.172704] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:09.173783] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:09.182609] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227709181431}) [2025-08-15 10:15:09.182625] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:09.182638] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:15:09.182888] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CEF-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.188725] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227709187502}) [2025-08-15 10:15:09.188739] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:09.188747] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=5][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:15:09.189931] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227709188755}) [2025-08-15 10:15:09.189955] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=19][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:09.189965] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:15:09.194475] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7830-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224108784023003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224108784023003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224108784023003, v:0}}], last_dump_ts_=1755224104163349, force_dump=false) [2025-08-15 10:15:09.194500] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7830-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.194630] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF7-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224108784023003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224108784023003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224108784023003, v:0}}], last_dump_ts_=1755224104163831, force_dump=false) [2025-08-15 10:15:09.194650] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF7-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.194754] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.194773] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A1-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.195082] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC029-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.195255] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C2-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.195948] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.197123] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:09.197151] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:09.197169] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.197180] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1003}, replicas:[{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:09.197290] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=85] build ls locality stat map(tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:15:09.197300] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.197311] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.197320] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:09.197330] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:09.197423] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=65] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.197425] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9A4-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.197444] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.197451] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.197458] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:09.197465] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3864, tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:15:09.197474] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:09.197801] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7830-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.197882] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF7-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224108784023003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224108784023002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224108784023003, v:0}}], last_dump_ts_=1755224104166787, force_dump=false) [2025-08-15 10:15:09.197904] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF7-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.197979] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A1-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.198294] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC029-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.198465] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C2-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.198720] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94361, tenant_id=1018, acc_dr_task=0) [2025-08-15 10:15:09.199880] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF7-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.199969] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF7-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.200063] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A1-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.200220] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A1-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.200335] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC029-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.200414] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7830-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.200448] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC029-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.200546] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C2-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.200602] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7830-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.200656] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C2-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.200788] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A1-0-0] [lt=11] get number(ret=0, service_type_=0, range=1, base_id=1755224109199808000, start_id=1755224109199808000, end_id=1755224109199808001) [2025-08-15 10:15:09.201542] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A1-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224109199808000, v:0}, tenant_id=1008) [2025-08-15 10:15:09.260301] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7620-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224108784023002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224109168184000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224109168184000, v:0}}], last_dump_ts_=1755224104230059, force_dump=false) [2025-08-15 10:15:09.260323] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7620-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.263260] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7620-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224109168184000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224109168184000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224109168184000, v:0}}], last_dump_ts_=1755224104233151, force_dump=false) [2025-08-15 10:15:09.263282] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7620-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.265171] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7620-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.265255] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7620-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.319566] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A09-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.319639] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882A-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.322093] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A09-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:09.322112] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A09-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.322186] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882A-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:09.322204] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.415710] INFO [RS] finish (ob_mview_maintenance_task.cpp:280) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=12] mview maintenace task finish(tenant_id_=1, round_=121919, status_=100, error_code_=0, cost_us_=20001339, prepare_cost_us_=0, gc_mview_cost_us_=1198, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:15:09.438730] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C684E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.457106] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E39B-0-0] [lt=12] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}) [2025-08-15 10:15:09.457151] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E39B-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=50, ddl_arg=NULL) [2025-08-15 10:15:09.458660] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:09.458670] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:09.458678] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:09.459981] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:09.481162] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A1-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:09.483673] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A1-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:09.498442] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E704-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.498537] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF095-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.530609] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:09.530620] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:09.530625] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:09.531286] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:09.531303] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:09.531313] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:09.531907] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:09.536745] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:09.536756] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:09.536761] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:09.543134] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:09.543146] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:09.543151] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:09.548051] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:09.548068] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:09.548074] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:09.549458] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:09.599862] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9583-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.602366] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9583-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:09.602385] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9583-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.605608] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5238-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:09.606599] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C3-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:09.608076] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5238-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:09.609037] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C3-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:09.615224] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:09.616454] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224109615151, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:15:09.616480] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224114616479, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:15:09.617099] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884F-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.618378] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8351-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224109285010002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224109285010002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224109285010002, v:0}}], last_dump_ts_=1755224104588331, force_dump=false) [2025-08-15 10:15:09.618406] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8351-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.619662] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884F-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:09.619678] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD884F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.621410] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8351-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.623308] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8351-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.623411] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8351-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:09.639273] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC62C-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.645158] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB16-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.647741] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB16-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:09.647761] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB16-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:09.677613] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D7-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:09.678121] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB866E-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:09.680190] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D7-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:09.680626] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB866E-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:09.687357] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6851-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.697682] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E1-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224109697095000, start_id=1755224109697095000, end_id=1755224109697095001) [2025-08-15 10:15:09.698396] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E1-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224109697095000, v:0}, tenant_id=1003) [2025-08-15 10:15:09.698638] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD63-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.704752] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:09.704765] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:09.704770] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:09.747968] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD72-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.747984] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF03C-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.767988] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F7D-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:09.770497] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F7D-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:09.806797] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6281-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:09.809319] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6281-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:09.836701] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5BE-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224109835957000, start_id=1755224109835957000, end_id=1755224109835957001) [2025-08-15 10:15:09.837482] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5BE-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224109835957000, v:0}, tenant_id=1004) [2025-08-15 10:15:09.845727] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CF2-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.862624] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B34C-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:09.897153] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224109544777, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224107782071004, replayable_scn:1755224107782071004, readable_scn:1755224107782071004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:09.988962] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58BD-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.989630] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9A0-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:09.991923] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17740][T1_TimerWK3_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=16] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:10.101443] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58B7-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.133731] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF098-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.134454] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF48-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.134840] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F0-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.136975] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF48-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:10.136993] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF48-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.137357] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F0-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:10.137374] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F0-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.146212] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF039-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.169110] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2BF-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224110168300000, start_id=1755224110168300000, end_id=1755224110168300001) [2025-08-15 10:15:10.169822] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2BF-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224110168300000, v:0}, tenant_id=1007) [2025-08-15 10:15:10.182923] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CF0-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.196780] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727239][T1015_DRService][T1015][YB420ACB0122-000633A351E5562B-0-0] [lt=16] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1015, thread_idx=1) [2025-08-15 10:15:10.197501] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9A5-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.200477] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [727239][T1015_DRService][T1015][YB420ACB0122-000633A351E5562C-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2413, tenant_id=1015, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:10.200984] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF8-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.201183] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A2-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.201419] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.201503] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7831-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.201535] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C3-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.202855] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [727239][T1015_DRService][T1015][YB420ACB0122-000633A351E5562C-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2357, tenant_id=1016, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:10.204047] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF8-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.204560] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02A-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.204641] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7831-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.204741] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A2-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.204802] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C3-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.205977] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF8-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.206099] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF8-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.206633] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7831-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.206663] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.206732] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7831-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.206765] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02A-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.206771] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C3-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.206778] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A2-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.206857] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C3-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.206901] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A2-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.266234] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7621-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.267725] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B632F-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}) [2025-08-15 10:15:10.267767] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B632F-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:10.269162] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7621-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.271099] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7621-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.271201] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7621-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.320295] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [438021][T1006_TimerWK0_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=13] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:10.320329] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [438021][T1006_TimerWK0_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=24] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:10.320347] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [438021][T1006_TimerWK0_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=13] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:10.321592] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [438024][T1006_TimerWK3_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=13] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:10.324949] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0A-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.324980] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882B-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.327477] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0A-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:10.327494] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0A-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.327611] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882B-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:10.327627] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882B-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.336946] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [17712][T1_BackupDataSr][T1][YB420ACB0122-0006337F03F7B80E-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:10.438803] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C684F-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.458697] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:10.458711] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:10.458716] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:10.489126] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A2-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:10.492776] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A2-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:10.498515] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E705-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.498611] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF096-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.502517] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF206-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.517231] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6985-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}) [2025-08-15 10:15:10.517267] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6985-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=39, ddl_arg=NULL) [2025-08-15 10:15:10.530387] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224108961765004, replayable_scn:1755224108961765004, readable_scn:1755224108961765004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224110526894, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224108961765004, replayable_scn:1755224108961765004, readable_scn:1755224108961765004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224110526894, ora_rowscn_=1755224110268154000, content_changed=true) [2025-08-15 10:15:10.530583] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:10.530595] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:10.530600] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:10.531364] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:10.531375] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:10.531380] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:10.537168] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224108784023003, replayable_scn:1755224108784023003, readable_scn:1755224108784023003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224110534267, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224108784023003, replayable_scn:1755224108784023003, readable_scn:1755224108784023003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224110534267, ora_rowscn_=1755224110202894000, content_changed=true) [2025-08-15 10:15:10.537359] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:10.537371] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:10.537376] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:10.542860] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:10.542871] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:10.542877] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:10.548084] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:10.548096] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:10.548101] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:10.590351] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17716][T1_DRService][T1][YB420ACB0122-0006337F0497B79F-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1, thread_idx=1) [2025-08-15 10:15:10.593948] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [17716][T1_DRService][T1][YB420ACB0122-0006337F0497B7A0-0-0] [lt=16] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2333, tenant_id=1, need_clean_task=true, service_epoch_to_check=2) [2025-08-15 10:15:10.605076] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9584-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.607557] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9584-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:10.607577] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9584-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.610117] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:10.611337] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224110610045, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:15:10.611365] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224115611363, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:15:10.613287] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5239-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:10.614220] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C4-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:10.615781] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5239-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:10.616672] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C4-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:10.622365] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8850-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.624449] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8352-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.624963] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8850-0-0] [lt=73] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:10.624981] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8850-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.627520] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8352-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.629477] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8352-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.629576] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8352-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:10.650520] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB17-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.652083] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438021][T1006_TimerWK0_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=8] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:10.653206] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB17-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:10.653225] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB17-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:10.685701] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D8-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:10.685878] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB866F-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:10.687432] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6852-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.688342] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D8-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:10.688378] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB866F-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:10.698715] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD64-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.704935] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:10.704958] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:10.704964] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:10.704976] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_time_us=190, return_code_array=[0, 0, 0], broadcast_times=2) [2025-08-15 10:15:10.748042] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD73-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.748056] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF03D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.775861] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F7E-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:10.778399] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F7E-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:10.814776] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6282-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:10.817324] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6282-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:10.845765] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CF3-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.864260] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B34D-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:10.924942] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:10.926131] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224110916105, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:15:10.926159] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224115926158, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:15:10.938111] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:10.939342] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224110938043, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:15:10.939368] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224115939367, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:15:10.939385] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224115939382, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:10.989039] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58BE-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:10.989704] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9A1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.040821] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CE-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1015, thread_idx=0) [2025-08-15 10:15:11.042101] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1015, only_for_display=false) [2025-08-15 10:15:11.043312] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:11.057096] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.058247] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.059381] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.059403] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1015, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:11.059417] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.059427] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1015, ls_id:{id:1}, replicas:[{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:11.059530] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=80] build ls locality stat map(tenant_id=1015, ls_id={id:1}) [2025-08-15 10:15:11.059537] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.059550] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.059561] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.059573] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:11.059651] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.059661] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.059670] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.059676] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.059685] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3756, tenant_id=1015, ls_id={id:1}) [2025-08-15 10:15:11.059695] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:11.060749] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18649, tenant_id=1015, acc_dr_task=0) [2025-08-15 10:15:11.061969] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1016, only_for_display=false) [2025-08-15 10:15:11.063194] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:11.076964] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.078298] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.079431] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.079454] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:11.079468] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.079479] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1}, replicas:[{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:11.079582] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=80] build ls locality stat map(tenant_id=1016, ls_id={id:1}) [2025-08-15 10:15:11.079591] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.079604] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.079614] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.079626] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:11.079703] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.079718] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.079727] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.079732] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.079741] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3942, tenant_id=1016, ls_id={id:1}) [2025-08-15 10:15:11.079751] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:11.080883] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:11.101505] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58B8-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.101526] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.102671] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.103808] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.103831] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:11.103847] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.103859] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1001}, replicas:[{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:11.103967] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=85] build ls locality stat map(tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:15:11.103976] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.103989] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.104001] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.104011] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:11.104090] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.104104] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.104115] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.104120] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.104124] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3731, tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:15:11.104135] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:11.105228] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:11.126072] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.127218] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.128371] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.128393] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:11.128410] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.128422] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1002}, replicas:[{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:11.128525] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=81] build ls locality stat map(tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:15:11.128532] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.128545] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.128554] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.128566] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:11.128643] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.128657] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.128666] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.128671] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.128680] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3760, tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:15:11.128690] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:11.129756] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:11.133804] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF099-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.139660] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF49-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.140040] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F1-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.142234] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF49-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:11.142251] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF49-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.142605] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F1-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:11.142624] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F1-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.146284] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF03A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.150561] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.151704] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.152869] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:11.152892] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:11.152906] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.152915] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1003}, replicas:[{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:11.153027] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=88] build ls locality stat map(tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:15:11.153034] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.153045] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.153053] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:11.153061] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:11.153137] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.153148] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.153156] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.153162] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:11.153169] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3736, tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:15:11.153177] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:11.154249] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93485, tenant_id=1016, acc_dr_task=0) [2025-08-15 10:15:11.177645] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:11.178863] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224111177593, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:15:11.178886] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224116178884, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:15:11.179951] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:11.181121] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224111179878, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:15:11.181140] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224116181139, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:11.182999] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CF1-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.197573] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9A6-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.207072] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF9-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.207703] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02B-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.207705] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7832-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.207797] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C4-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.207822] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A3-0-0] [lt=10] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224110786704004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224110786704004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224110786704004, v:0}}], last_dump_ts_=1755224106176807, force_dump=false) [2025-08-15 10:15:11.207848] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A3-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.210227] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.210720] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7832-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.210745] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02B-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224110786704003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224110786704003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224110786704003, v:0}}], last_dump_ts_=1755224106179306, force_dump=false) [2025-08-15 10:15:11.210768] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02B-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.210803] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C4-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.210875] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A3-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224110786704004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224110786704004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224110786704004, v:0}}], last_dump_ts_=1755224106179823, force_dump=false) [2025-08-15 10:15:11.210897] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A3-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.212209] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF9-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.212324] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BF9-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.212677] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7832-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.212788] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02B-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.212830] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7832-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.212854] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C4-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.212882] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A3-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.212887] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02B-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.212933] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C4-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.212986] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A3-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.272141] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7622-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.275272] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7622-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.277230] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7622-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.277314] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7622-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.330182] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0B-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.330313] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882C-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.332766] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0B-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:11.332784] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.332863] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882C-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:11.332881] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.379183] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E4C-0-0] [lt=17] get number(ret=0, service_type_=0, range=1, base_id=1755224111378956000, start_id=1755224111378956000, end_id=1755224111378956001) [2025-08-15 10:15:11.379966] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E4C-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224111378956000, v:0}, tenant_id=1018) [2025-08-15 10:15:11.438876] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6850-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.457543] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98CC8-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}) [2025-08-15 10:15:11.457587] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98CC8-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:15:11.458589] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224109283964004, replayable_scn:1755224109283964004, readable_scn:1755224109283964004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224111455751, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224109283964004, replayable_scn:1755224109283964004, readable_scn:1755224109283964004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224111455751, ora_rowscn_=1755224110625848000, content_changed=true) [2025-08-15 10:15:11.458769] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:11.458781] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:11.458786] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:11.498126] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A3-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:11.498586] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E706-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.498681] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF097-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.500678] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A3-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:11.530738] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:11.530756] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:11.530768] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:11.531252] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224109852767002, replayable_scn:1755224109852767002, readable_scn:1755224109852767002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224111528390, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224109852767002, replayable_scn:1755224109852767002, readable_scn:1755224109852767002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224111528390, ora_rowscn_=1755224111209724000, content_changed=true) [2025-08-15 10:15:11.531436] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:11.531446] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:11.531454] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:11.536845] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:11.536856] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:11.536862] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:11.542763] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224109784424004, replayable_scn:1755224109784424004, readable_scn:1755224109784424004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224111539978, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224109784424004, replayable_scn:1755224109784424004, readable_scn:1755224109784424004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224111539978, ora_rowscn_=1755224111208670000, content_changed=true) [2025-08-15 10:15:11.542928] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:11.542947] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:11.542953] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:11.547997] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224109785480003, replayable_scn:1755224109785480003, readable_scn:1755224109785480003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224111545253, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224109785480003, replayable_scn:1755224109785480003, readable_scn:1755224109785480003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224111545253, ora_rowscn_=1755224111209724000, content_changed=true) [2025-08-15 10:15:11.548143] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:11.548153] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:11.548161] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:11.610297] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9585-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.612781] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9585-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:11.612798] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9585-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.621062] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD523A-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:11.621853] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C5-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:11.623543] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD523A-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:11.624344] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C5-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:11.627739] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8851-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.630282] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8851-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:11.630302] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8851-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.630541] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8353-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.633507] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8353-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.635405] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8353-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.635506] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8353-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:11.644154] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC62D-0-0] [lt=19] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.656046] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB18-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.658597] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB18-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:11.658616] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB18-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:11.687508] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6853-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.693629] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D9-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:11.693655] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8670-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:11.696120] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52D9-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:11.696132] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8670-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:11.698805] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD65-0-0] [lt=25] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.704797] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224109837010004, replayable_scn:1755224109837010004, readable_scn:1755224109837010004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224111701858, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224109837010004, replayable_scn:1755224109837010004, readable_scn:1755224109837010004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224111701858, ora_rowscn_=1755224111209724000, content_changed=true) [2025-08-15 10:15:11.704992] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:11.705012] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:11.705019] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:11.748114] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD74-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.748128] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF03E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.755117] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475910][T1008_TimerWK2_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:11.768775] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709535][T1014_TimerWK2_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:11.774212] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7F9C-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224111773425000, start_id=1755224111773425000, end_id=1755224111773425001) [2025-08-15 10:15:11.774982] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7F9C-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224111773425000, v:0}, tenant_id=1014) [2025-08-15 10:15:11.783783] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F7F-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:11.786268] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F7F-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:11.819201] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:11.820415] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224111819129, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:15:11.820439] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224116820437, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:15:11.822744] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6283-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:11.825357] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6283-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:11.845837] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CF4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.865689] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B34E-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:11.989112] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58BF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:11.989779] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9A2-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.090376] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727135][T1016_TimerWK3_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:12.101581] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58B9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.122950] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E5-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1013, thread_idx=0) [2025-08-15 10:15:12.124229] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=20] start try tenant disaster recovery(tenant_id=1013, only_for_display=false) [2025-08-15 10:15:12.125484] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:12.133874] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF09A-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.139760] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.140966] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.142182] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.142205] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1013, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:12.142223] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.142235] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1013, ls_id:{id:1}, replicas:[{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:12.142346] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=86] build ls locality stat map(tenant_id=1013, ls_id={id:1}) [2025-08-15 10:15:12.142354] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.142367] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.142378] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.142389] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:12.142469] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.142483] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.142492] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.142499] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.142508] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3944, tenant_id=1013, ls_id={id:1}) [2025-08-15 10:15:12.142518] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:12.143826] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19597, tenant_id=1013, acc_dr_task=0) [2025-08-15 10:15:12.145010] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4A-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.145284] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F2-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.145357] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=18] start try tenant disaster recovery(tenant_id=1014, only_for_display=false) [2025-08-15 10:15:12.146356] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF03B-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.146578] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:12.147562] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4A-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:12.147580] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.147784] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F2-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:12.147801] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F2-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.160789] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.162002] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.163223] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.163246] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:12.163264] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.163277] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1}, replicas:[{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:12.163386] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=85] build ls locality stat map(tenant_id=1014, ls_id={id:1}) [2025-08-15 10:15:12.163394] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.163407] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.163418] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.163427] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:12.163510] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.163524] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.163533] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.163538] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.163546] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3958, tenant_id=1014, ls_id={id:1}) [2025-08-15 10:15:12.163557] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:12.164618] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:12.183070] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CF2-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.186015] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.187185] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.188368] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.188391] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:12.188409] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.188421] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1001}, replicas:[{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:12.188526] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=82] build ls locality stat map(tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:15:12.188534] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.188547] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.188553] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.188564] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:12.188641] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.188657] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.188670] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=10] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.188681] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.188686] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3847, tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:15:12.188696] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:12.189794] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:12.194246] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=16][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227712193058}) [2025-08-15 10:15:12.194265] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:12.194278] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:15:12.194662] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727957][T1018_TimerWK1_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=18] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:12.197646] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9A7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.200227] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227712199060}) [2025-08-15 10:15:12.200244] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:12.200257] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:15:12.201464] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227712200269}) [2025-08-15 10:15:12.201480] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:12.201491] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:15:12.211868] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.213105] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.213415] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFA-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.213830] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7833-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.213919] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C5-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.213978] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02C-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224111788157003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224111788157003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224111788157003, v:0}}], last_dump_ts_=1755224107182382, force_dump=false) [2025-08-15 10:15:12.214002] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.214034] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A4-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.214307] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.214331] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:12.214347] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.214360] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1002}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:12.214466] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=82] build ls locality stat map(tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:15:12.214474] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.214488] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.214498] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.214510] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:12.214587] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.214601] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.214611] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.214616] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.214625] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3940, tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:15:12.214631] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:12.215873] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:12.216653] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFA-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.217041] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C5-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.217085] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7833-0-0] [lt=20] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224111788157003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224111788157003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224111788157003, v:0}}], last_dump_ts_=1755224107185210, force_dump=false) [2025-08-15 10:15:12.217110] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7833-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.217207] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.217227] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A4-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.218647] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFA-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.218758] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFA-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.219131] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C5-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.219188] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.219217] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C5-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.219251] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7833-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.219254] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A4-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.219305] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.219341] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7833-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.219354] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A4-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.237229] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.238377] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.239526] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:12.239549] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:12.239566] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.239577] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1003}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:12.239681] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=81] build ls locality stat map(tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:15:12.239690] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.239703] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.239711] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:12.239721] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:12.239797] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.239811] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.239820] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.239827] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:12.239836] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3756, tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:15:12.239846] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:12.240902] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=97051, tenant_id=1014, acc_dr_task=0) [2025-08-15 10:15:12.248946] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619487][T1012_TimerWK2_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=7] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=10) [2025-08-15 10:15:12.268264] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BAF-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}) [2025-08-15 10:15:12.268307] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BAF-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:15:12.278230] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7623-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.281189] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7623-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.283083] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7623-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.283162] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7623-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.308963] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487605][T1004_TimerWK2_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:12.323787] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:12.325073] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224112323723, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:15:12.325096] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224117325095, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:15:12.335516] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0C-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.335579] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882D-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.338013] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0C-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:12.338031] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.338107] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882D-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:12.338124] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.410141] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:12.411410] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224112410072, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:15:12.411434] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224117411433, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:15:12.438409] INFO [RS] gc_mview (ob_mview_maintenance_task.cpp:267) [475908][T1008_TimerWK0_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=12] mview maintenance task gc mview success(tenant_id_=1008, round_=105550, gc_mview_cost_us_=1238, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:15:12.438953] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6851-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.458847] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:12.458859] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:12.458865] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:12.468125] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728063][T1017_DRService][T1017][YB420ACB0122-000633A358855625-0-0] [lt=13] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1017, thread_idx=1) [2025-08-15 10:15:12.471800] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [728063][T1017_DRService][T1017][YB420ACB0122-000633A358855626-0-0] [lt=12] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2356, tenant_id=1017, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:12.474139] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [728063][T1017_DRService][T1017][YB420ACB0122-000633A358855626-0-0] [lt=14] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2319, tenant_id=1018, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:12.498660] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E707-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.498719] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF098-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.502800] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF207-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.506050] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A4-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:12.508542] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A4-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:12.517720] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D57E1-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80273438GB}) [2025-08-15 10:15:12.517761] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D57E1-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2025-08-15 10:15:12.530811] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:12.530823] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:12.530830] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:12.531436] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:12.531447] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:12.531454] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:12.532735] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:12.536876] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:12.536886] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:12.536894] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:12.542982] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:12.542992] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:12.543004] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:12.544290] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:12.548204] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:12.548220] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:12.548232] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:12.615546] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9586-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.618080] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9586-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:12.618098] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9586-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.628861] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD523B-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:12.629576] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C6-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:12.632121] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C6-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:12.632515] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD523B-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:12.633054] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8852-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.635593] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8852-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:12.635612] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8852-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.636541] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8354-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.639384] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:340) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8354-0-0] [lt=14] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1014, ls_id:{id:1}, sync_scn:{val:1755224112289250001, v:0}, readable_scn:{val:1755224112289250001, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-08-15 10:15:12.639721] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8354-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.641742] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8354-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.641847] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8354-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:12.641886] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:459) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8354-0-0] [lt=7] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1755224110285017004, v:0}) [2025-08-15 10:15:12.661413] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB19-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.663991] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB19-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:12.664009] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB19-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:12.687584] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6854-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.698878] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD66-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.701516] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52DA-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:12.701530] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8671-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:12.704024] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52DA-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:12.704025] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8671-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:12.705018] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:12.705033] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:12.705039] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:12.748187] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD75-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.748199] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF03F-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.788555] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:12.789773] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224112788484, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:15:12.789796] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224117789795, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:15:12.790333] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:12.791493] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224112790263, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:15:12.791514] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224117791513, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:15:12.791657] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F80-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:12.795397] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F80-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:12.830796] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6284-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:12.833318] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6284-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:12.845913] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CF5-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.867164] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B34F-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:12.898776] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224112544491, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224110786704003, replayable_scn:1755224110786704003, readable_scn:1755224110786704003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:12.951965] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:12.953215] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224112951883, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:15:12.953239] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224117953237, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:15:12.989190] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58C0-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:12.989856] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9A3-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.037717] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:13.039001] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224113037645, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:15:13.039023] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224118039022, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:15:13.101659] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58BA-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.133945] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF09B-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.146427] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF03C-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.150371] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4B-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.150480] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F3-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.152982] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F3-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:13.152999] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4B-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:13.153000] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F3-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.153018] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4B-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.183144] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CF3-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.197720] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9A8-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.219788] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFB-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.220198] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C6-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224112790335003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224112790335003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224112790335003, v:0}}], last_dump_ts_=1755224108188802, force_dump=false) [2025-08-15 10:15:13.220225] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C6-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.220297] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02D-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.220311] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7834-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.220351] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A5-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.222912] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFB-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.223327] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C6-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224112790335003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224112790335003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224112790335003, v:0}}], last_dump_ts_=1755224108192057, force_dump=false) [2025-08-15 10:15:13.223342] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7834-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.223354] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C6-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.223471] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A5-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.223505] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02D-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.224965] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFB-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.225080] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFB-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.225420] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C6-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.225421] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7834-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.225474] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A5-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.225523] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7834-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.225532] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C6-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.225570] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A5-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.225583] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.225670] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.284343] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7624-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.287466] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7624-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.289426] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7624-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.289511] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7624-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.340751] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0D-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.340821] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.343280] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0D-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:13.343298] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.343410] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882E-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:13.343428] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.439029] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6852-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.457986] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A06E1-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}) [2025-08-15 10:15:13.458031] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A06E1-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:15:13.458894] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:13.458908] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:13.458916] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:13.498735] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E708-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.498789] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF099-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.513829] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A5-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:13.516333] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A5-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:13.530685] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224111966508004, replayable_scn:1755224111966508004, readable_scn:1755224111966508004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224113527580, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224111966508004, replayable_scn:1755224111966508004, readable_scn:1755224111966508004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224113527580, ora_rowscn_=1755224113286646000, content_changed=true) [2025-08-15 10:15:13.530863] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:13.530873] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:13.530880] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:13.531513] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:13.531524] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:13.531532] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:13.536958] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:13.536969] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:13.536974] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:13.538248] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:13.543065] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:13.543075] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:13.543084] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:13.545774] INFO [RS.LB] idle (ob_root_balancer.cpp:87) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=31] root balance idle(idle_time=10000000) [2025-08-15 10:15:13.545791] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:70) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=11] start check unit migrate finish [2025-08-15 10:15:13.545806] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=9] try check migrate unit finish by tenant(tenant_id=1) [2025-08-15 10:15:13.548276] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:13.548288] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:13.548294] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:13.555155] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1004) [2025-08-15 10:15:13.580865] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1006) [2025-08-15 10:15:13.595727] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:13.596969] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224113595651, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:15:13.596993] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224118596992, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:15:13.606898] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1008) [2025-08-15 10:15:13.620806] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9587-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.623385] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9587-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:13.623407] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9587-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.632857] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1012) [2025-08-15 10:15:13.637465] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C7-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:13.637769] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD523C-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:13.638389] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8853-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.639982] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C7-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:13.640273] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD523C-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:13.641071] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8853-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:13.641086] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8853-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.642884] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8355-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.646154] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8355-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224113289809003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224113289809003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224113289809003, v:0}}], last_dump_ts_=1755224108615346, force_dump=false) [2025-08-15 10:15:13.646178] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8355-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.648198] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8355-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.648368] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8355-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:13.649070] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC62E-0-0] [lt=13] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.659009] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1014) [2025-08-15 10:15:13.666877] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1A-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.669562] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1A-0-0] [lt=74] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:13.669582] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1A-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:13.685244] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=11] try check migrate unit finish by tenant(tenant_id=1016) [2025-08-15 10:15:13.687662] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6855-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.698953] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD67-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.705669] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:13.705689] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:13.705696] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:13.705705] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_time_us=231, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:13.707067] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:13.709426] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8672-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:13.711697] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52DB-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:13.711997] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8672-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:13.714351] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1018) [2025-08-15 10:15:13.717078] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52DB-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:13.741277] INFO [RS] check (ob_alter_locality_finish_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149C-0-0] [lt=18] start to check alter locality finish [2025-08-15 10:15:13.742448] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149D-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.743591] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149D-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.745889] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149F-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.747054] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8149F-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.748259] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD76-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.748269] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF040-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.749317] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814A1-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.750481] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814A1-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.752775] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814A3-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.753948] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814A3-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.756266] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814A5-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.757433] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814A5-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.759720] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814A7-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.760861] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814A7-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.763160] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814A9-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.764317] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814A9-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.766588] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AB-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.767728] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AB-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.768916] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AB-0-0] [lt=13] start check shrink resource pool [2025-08-15 10:15:13.768934] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AC-0-0] [lt=11] start check shrink resource pool(tenant_id=1) [2025-08-15 10:15:13.770065] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AC-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.771219] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AC-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.771272] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AD-0-0] [lt=12] start check shrink resource pool(tenant_id=1004) [2025-08-15 10:15:13.772391] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AD-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.773534] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AD-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.773586] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AE-0-0] [lt=12] start check shrink resource pool(tenant_id=1006) [2025-08-15 10:15:13.774704] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AE-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.775832] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AE-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.775883] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AF-0-0] [lt=12] start check shrink resource pool(tenant_id=1008) [2025-08-15 10:15:13.777015] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AF-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.778162] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814AF-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.778220] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B0-0-0] [lt=12] start check shrink resource pool(tenant_id=1012) [2025-08-15 10:15:13.779345] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B0-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.780497] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B0-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.780548] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B1-0-0] [lt=12] start check shrink resource pool(tenant_id=1014) [2025-08-15 10:15:13.781673] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B1-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.782808] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B1-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.782856] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B2-0-0] [lt=11] start check shrink resource pool(tenant_id=1016) [2025-08-15 10:15:13.783985] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B2-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.785119] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B2-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.785169] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B3-0-0] [lt=13] start check shrink resource pool(tenant_id=1018) [2025-08-15 10:15:13.786285] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B3-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.787427] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B3-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.787479] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:99) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B3-0-0] [lt=12] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.787492] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B3-0-0] [lt=10] start check primary zone [2025-08-15 10:15:13.787505] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B4-0-0] [lt=7] start check primary zone(tenant_id=1) [2025-08-15 10:15:13.788613] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B4-0-0] [lt=6][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.788662] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B5-0-0] [lt=12] start check primary zone(tenant_id=1004) [2025-08-15 10:15:13.789773] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B5-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.789823] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B6-0-0] [lt=12] start check primary zone(tenant_id=1006) [2025-08-15 10:15:13.790970] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B6-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.791021] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B7-0-0] [lt=12] start check primary zone(tenant_id=1008) [2025-08-15 10:15:13.792150] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B7-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.792203] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B8-0-0] [lt=12] start check primary zone(tenant_id=1012) [2025-08-15 10:15:13.793345] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B8-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.793396] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B9-0-0] [lt=13] start check primary zone(tenant_id=1014) [2025-08-15 10:15:13.794492] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814B9-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.794541] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BA-0-0] [lt=12] start check primary zone(tenant_id=1016) [2025-08-15 10:15:13.795667] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BA-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.795724] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=14] start check primary zone(tenant_id=1018) [2025-08-15 10:15:13.796823] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:13.796874] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:166) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=12] start do unit balance [2025-08-15 10:15:13.800776] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F81-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:13.803212] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:350) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=9] start distribute for server status change [2025-08-15 10:15:13.803314] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:401) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=13] finish distribute for server status change(ret=0) [2025-08-15 10:15:13.803318] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F81-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:13.803324] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:200) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=8] finish do unit balance(ret=0) [2025-08-15 10:15:13.809346] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:115) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=4] start to do tenant group unit balance [2025-08-15 10:15:13.809390] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=13] start to do tenant group unit balance(zone=zone1) [2025-08-15 10:15:13.809405] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=9] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.203.1.34:2882", disk_in_use_:3126853632, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:13.809423] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=12] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:13.809454] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:13.809472] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=9] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:13.809483] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=8] start do non-tenantgroup unit balance task(zone=zone1) [2025-08-15 10:15:13.809490] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=5] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:13.809514] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=14] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.1.34:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:13.809560] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=36] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.203.1.34:2882", disk_in_use_:3126853632, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:13.809568] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=6] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.1.34:2882", disk_in_use_:3126853632, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:13.809582] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:13.809594] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=9] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1006, unit_config:{unit_config_id:1005, name:"config_opsone_zone1_s_2_4_ywx", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7988, unit_config:0x7f1e9aa71c38, pool:0x7f1c3beb2f18}, {unit_id:1008, tenant_id:1006, resource_pool_id:1008, unit_config:{unit_config_id:1008, name:"config_test_zone1_s_2_4_emp", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f77d8, unit_config:0x7f1e9aa71998, pool:0x7f1c3beb27a8}, {unit_id:1011, tenant_id:1008, resource_pool_id:1011, unit_config:{unit_config_id:1011, name:"config_mcp_zone1_s_4_8_tmc", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7550, unit_config:0x7f1e9aa716f8, pool:0x7f1e443fda40}, {unit_id:1016, tenant_id:1012, resource_pool_id:1016, unit_config:{unit_config_id:1017, name:"config_oms_zone1_s_4_8_gzp", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f73a0, unit_config:0x7f1e9aa71458, pool:0x7f1e443fd2d0}, {unit_id:1019, tenant_id:1014, resource_pool_id:1019, unit_config:{unit_config_id:1019, name:"config_grafana_test_zone1_s_1_2_pri", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7118, unit_config:0x7f1e9aa71298, pool:0x7f1e443fc7a8}, {unit_id:1024, tenant_id:1016, resource_pool_id:1024, unit_config:{unit_config_id:1023, name:"config_temporalio_zone1_s_2_4_ykw", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6ce0, unit_config:0x7f1e9aa70f18, pool:0x7f1e74dff2d0}, {unit_id:1026, tenant_id:1018, resource_pool_id:1026, unit_config:{unit_config_id:1027, name:"config_media_zone1_s_2_4_kmn", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6b30, unit_config:0x7f1e9aa70b98, pool:0x7f1e74dfeb60}], not_grant_units=[], available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:13.809654] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=35] start to do tenant group unit balance(zone=zone2) [2025-08-15 10:15:13.809663] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.203.4.38:2882", disk_in_use_:3110076416, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:13.809676] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=9] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:13.809697] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=12] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:13.809708] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=6] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:13.809713] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=4] start do non-tenantgroup unit balance task(zone=zone2) [2025-08-15 10:15:13.809722] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=6] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:13.809735] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.38:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:13.809767] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=25] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.203.4.38:2882", disk_in_use_:3110076416, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:13.809773] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=4] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.38:2882", disk_in_use_:3110076416, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:13.809783] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:13.809794] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=8] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1004, unit_config:{unit_config_id:1004, name:"config_opsone_zone2_s_2_4_wag", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7b38, unit_config:0x7f1e9aa71d18, pool:0x7f1c3beb3688}, {unit_id:1009, tenant_id:1006, resource_pool_id:1009, unit_config:{unit_config_id:1009, name:"config_test_zone2_s_2_4_yey", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7700, unit_config:0x7f1e9aa718b8, pool:0x7f1c3beb23f0}, {unit_id:1010, tenant_id:1008, resource_pool_id:1010, unit_config:{unit_config_id:1010, name:"config_mcp_zone2_s_4_8_bjb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7628, unit_config:0x7f1e9aa717d8, pool:0x7f1c3beb2038}, {unit_id:1018, tenant_id:1012, resource_pool_id:1018, unit_config:{unit_config_id:1016, name:"config_oms_zone2_s_4_8_kbx", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f71f0, unit_config:0x7f1e9aa71538, pool:0x7f1e443fcb60}, {unit_id:1021, tenant_id:1014, resource_pool_id:1021, unit_config:{unit_config_id:1021, name:"config_grafana_test_zone2_s_1_2_yyc", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f6f68, unit_config:0x7f1e9aa710d8, pool:0x7f1e443fc038}, {unit_id:1023, tenant_id:1016, resource_pool_id:1023, unit_config:{unit_config_id:1022, name:"config_temporalio_zone2_s_2_4_lsv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6db8, unit_config:0x7f1e9aa70ff8, pool:0x7f1e74dff688}, {unit_id:1027, tenant_id:1018, resource_pool_id:1027, unit_config:{unit_config_id:1025, name:"config_media_zone2_s_2_4_yzd", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6a58, unit_config:0x7f1e9aa70d58, pool:0x7f1e74dfe7a8}], not_grant_units=[], available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:13.809854] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=37] start to do tenant group unit balance(zone=zone3) [2025-08-15 10:15:13.809864] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=7] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.203.4.39:2882", disk_in_use_:3009413120, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:13.809875] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:13.809892] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=9] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:13.809903] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=7] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:13.809913] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone3) [2025-08-15 10:15:13.809917] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:13.809930] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=8] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80273438GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.39:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80273438GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:13.809966] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=28] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.203.4.39:2882", disk_in_use_:3009413120, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:13.809972] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=4] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.39:2882", disk_in_use_:3009413120, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:13.809983] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:13.809993] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=8] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1005, unit_config:{unit_config_id:1006, name:"config_opsone_zone3_s_2_4_vxv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7a60, unit_config:0x7f1e9aa71b58, pool:0x7f1c3beb32d0}, {unit_id:1007, tenant_id:1006, resource_pool_id:1007, unit_config:{unit_config_id:1007, name:"config_test_zone3_s_2_4_mui", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f78b0, unit_config:0x7f1e9aa71a78, pool:0x7f1c3beb2b60}, {unit_id:1012, tenant_id:1008, resource_pool_id:1012, unit_config:{unit_config_id:1012, name:"config_mcp_zone3_s_4_8_seu", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7478, unit_config:0x7f1e9aa71618, pool:0x7f1e443fd688}, {unit_id:1017, tenant_id:1012, resource_pool_id:1017, unit_config:{unit_config_id:1018, name:"config_oms_zone3_s_4_8_nvb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f72c8, unit_config:0x7f1e9aa71378, pool:0x7f1e443fcf18}, {unit_id:1020, tenant_id:1014, resource_pool_id:1020, unit_config:{unit_config_id:1020, name:"config_grafana_test_zone3_s_1_2_xbz", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7040, unit_config:0x7f1e9aa711b8, pool:0x7f1e443fc3f0}, {unit_id:1022, tenant_id:1016, resource_pool_id:1022, unit_config:{unit_config_id:1024, name:"config_temporalio_zone3_s_2_4_rxz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6e90, unit_config:0x7f1e9aa70e38, pool:0x7f1e74dffa40}, {unit_id:1025, tenant_id:1018, resource_pool_id:1025, unit_config:{unit_config_id:1026, name:"config_media_zone3_s_2_4_ukz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6c08, unit_config:0x7f1e9aa70c78, pool:0x7f1e74dfef18}], not_grant_units=[], available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:13.838805] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6285-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:13.842631] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6285-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:13.845949] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CF6-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.868608] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B350-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:13.989262] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58C1-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:13.989934] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9A4-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.101730] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58BB-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.134021] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF09C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.146502] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF03D-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.155773] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4C-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.155784] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F4-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.158314] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F4-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:14.158357] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4C-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:14.158331] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F4-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.158377] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4C-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.183218] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CF4-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.197792] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9A9-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.226144] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFC-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224113790710004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224113790710005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224113790710005, v:0}}], last_dump_ts_=1755224109194630, force_dump=false) [2025-08-15 10:15:14.226171] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFC-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.226507] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C7-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.226508] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7835-0-0] [lt=9] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224113790710004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224113790710004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224113790710004, v:0}}], last_dump_ts_=1755224109194474, force_dump=false) [2025-08-15 10:15:14.226525] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A6-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.226534] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7835-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.226622] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.229283] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFC-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224113790710004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224113790710005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224113790710004, v:0}}], last_dump_ts_=1755224109197882, force_dump=false) [2025-08-15 10:15:14.229307] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFC-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.229582] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.229686] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7835-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.229765] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A6-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.229849] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C7-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.231267] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFC-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.231373] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFC-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.231576] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.231761] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A6-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.231767] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.231846] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A6-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.231921] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7835-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.232009] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C7-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.232076] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7835-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.232167] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C7-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.268773] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B6330-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}) [2025-08-15 10:15:14.268814] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B6330-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:15:14.290585] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7625-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224113790710003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224113790710004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224113790710004, v:0}}], last_dump_ts_=1755224109260300, force_dump=false) [2025-08-15 10:15:14.290610] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7625-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.293621] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7625-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224113790710004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224113790710005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224113790710005, v:0}}], last_dump_ts_=1755224109263259, force_dump=false) [2025-08-15 10:15:14.293644] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7625-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.295546] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7625-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.295732] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7625-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.346092] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.346139] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882F-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.348628] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0E-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:14.348645] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0E-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.348689] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882F-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:14.348710] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D882F-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.439102] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6853-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.458854] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224112288196004, replayable_scn:1755224112288196004, readable_scn:1755224112288196004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224114455670, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224112288196004, replayable_scn:1755224112288196004, readable_scn:1755224112288196004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224114455670, ora_rowscn_=1755224113644375000, content_changed=true) [2025-08-15 10:15:14.459038] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:14.459049] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:14.459057] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:14.460413] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:14.498806] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E709-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.498818] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF09A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.503081] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF208-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.518208] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A63-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80273438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}) [2025-08-15 10:15:14.518247] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A63-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:15:14.521772] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A6-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:14.524287] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A6-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:14.529140] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C53-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1011, thread_idx=0) [2025-08-15 10:15:14.530394] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1011, only_for_display=false) [2025-08-15 10:15:14.530912] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:14.530924] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:14.530932] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:14.531407] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224112792405004, replayable_scn:1755224112792405004, readable_scn:1755224112792405004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224114528247, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224112792405004, replayable_scn:1755224112792405004, readable_scn:1755224112792405004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224114528247, ora_rowscn_=1755224114228395000, content_changed=true) [2025-08-15 10:15:14.531587] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:14.531596] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:14.531604] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:14.531620] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:14.532223] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:14.536833] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224112790335003, replayable_scn:1755224112790335003, readable_scn:1755224112790335003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224114534571, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224112790335003, replayable_scn:1755224112790335003, readable_scn:1755224112790335003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224114534571, ora_rowscn_=1755224114228395000, content_changed=true) [2025-08-15 10:15:14.536992] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:14.537003] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:14.537008] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:14.538384] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:104) [17458][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=11] start checking lost replicas [2025-08-15 10:15:14.542992] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224112789281005, replayable_scn:1755224112789281005, readable_scn:1755224112789281005, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224114539840, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224112789281005, replayable_scn:1755224112789281005, readable_scn:1755224112789281005, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224114539840, ora_rowscn_=1755224114228395000, content_changed=true) [2025-08-15 10:15:14.543163] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:14.543172] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:14.543180] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:14.545486] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.546660] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.547804] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.547826] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1011, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:14.547842] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.547853] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1011, ls_id:{id:1}, replicas:[{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:14.547964] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=86] build ls locality stat map(tenant_id=1011, ls_id={id:1}) [2025-08-15 10:15:14.547972] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.547986] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.547997] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.548009] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:14.548087] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.548102] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.548112] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.548118] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.548126] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3806, tenant_id=1011, ls_id={id:1}) [2025-08-15 10:15:14.548132] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:14.548222] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224112789281004, replayable_scn:1755224112789281004, readable_scn:1755224112789281004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224114545110, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224112789281004, replayable_scn:1755224112789281004, readable_scn:1755224112789281004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224114545110, ora_rowscn_=1755224114228395000, content_changed=true) [2025-08-15 10:15:14.548380] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:14.548391] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:14.548398] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:14.549218] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18824, tenant_id=1011, acc_dr_task=0) [2025-08-15 10:15:14.549691] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:14.550420] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1012, only_for_display=false) [2025-08-15 10:15:14.551601] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:14.565636] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.566821] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.567962] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.567982] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:14.567997] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.568006] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1}, replicas:[{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:14.568107] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=79] build ls locality stat map(tenant_id=1012, ls_id={id:1}) [2025-08-15 10:15:14.568115] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.568126] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.568134] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.568144] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:14.568219] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.568231] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.568238] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.568246] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.568252] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=4067, tenant_id=1012, ls_id={id:1}) [2025-08-15 10:15:14.568259] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:14.569330] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:14.590165] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.591335] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.592478] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.592499] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:14.592515] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.592524] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1001}, replicas:[{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:14.592623] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=77] build ls locality stat map(tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:15:14.592634] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.592645] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.592653] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.592662] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:14.592759] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=73] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.592771] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.592779] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.592785] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.592792] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3791, tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:15:14.592800] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:14.593866] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:14.615131] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.616293] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.616552] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:14.617447] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.617467] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:14.617480] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.617490] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1002}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:14.617592] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=79] build ls locality stat map(tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:15:14.617602] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.617613] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.617621] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.617630] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:14.617705] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.617717] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.617725] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.617731] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.617737] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3753, tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:15:14.617745] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:14.617777] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224114616479, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:15:14.617799] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224119617798, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:15:14.618809] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:14.626124] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9588-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.628679] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9588-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:14.628696] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9588-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.640026] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.641173] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.642317] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.642338] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:14.642353] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.642362] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1003}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:14.642462] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=77] build ls locality stat map(tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:15:14.642470] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.642482] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.642491] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.642504] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:14.642582] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.642595] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.642603] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.642610] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.642620] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3722, tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:15:14.642628] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:14.643794] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94562, tenant_id=1012, acc_dr_task=0) [2025-08-15 10:15:14.643816] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8854-0-0] [lt=42] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.645226] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C8-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:14.645541] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD523D-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:14.646388] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8854-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:14.646405] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8854-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.647667] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C8-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:14.647992] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD523D-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:14.649509] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8356-0-0] [lt=9] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224114291565002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224114291565003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224114291565003, v:0}}], last_dump_ts_=1755224109618377, force_dump=false) [2025-08-15 10:15:14.649536] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8356-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.652573] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8356-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.654509] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8356-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.654595] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8356-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:14.672405] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1B-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.675042] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1B-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:14.675061] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1B-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:14.687727] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6856-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.699032] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD68-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.704920] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224112789281004, replayable_scn:1755224112789281004, readable_scn:1755224112789281004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224114701886, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224112789281004, replayable_scn:1755224112789281004, readable_scn:1755224112789281004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224114701886, ora_rowscn_=1755224114228395000, content_changed=true) [2025-08-15 10:15:14.705107] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=25] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:14.705120] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:14.705126] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:14.714464] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476036][T1007_DRService][T1007][YB420ACB0122-0006338215870D0D-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1007, thread_idx=1) [2025-08-15 10:15:14.717365] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8673-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:14.718039] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [476036][T1007_DRService][T1007][YB420ACB0122-0006338215870D0E-0-0] [lt=13] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2351, tenant_id=1007, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:14.719881] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8673-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:14.720372] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [476036][T1007_DRService][T1007][YB420ACB0122-0006338215870D0E-0-0] [lt=13] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2314, tenant_id=1008, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:14.722389] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52DC-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:14.724949] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52DC-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:14.748332] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD77-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.748340] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF041-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.808738] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F82-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:14.811302] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F82-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:14.846023] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CF7-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.848059] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6286-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:14.850588] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6286-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:14.869995] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B351-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:14.871318] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED1-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1007, thread_idx=0) [2025-08-15 10:15:14.872573] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1007, only_for_display=false) [2025-08-15 10:15:14.873801] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:14.887738] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.888982] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.890169] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.890192] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1007, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:14.890209] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.890218] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1007, ls_id:{id:1}, replicas:[{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:14.890319] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=79] build ls locality stat map(tenant_id=1007, ls_id={id:1}) [2025-08-15 10:15:14.890327] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.890340] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.890346] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.890355] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:14.890430] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.890443] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.890451] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.890458] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.890462] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3904, tenant_id=1007, ls_id={id:1}) [2025-08-15 10:15:14.890470] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:14.891582] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19009, tenant_id=1007, acc_dr_task=0) [2025-08-15 10:15:14.892803] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1008, only_for_display=false) [2025-08-15 10:15:14.894032] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:14.907899] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.909082] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.910245] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.910267] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:14.910287] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.910297] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1}, replicas:[{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:14.910401] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=81] build ls locality stat map(tenant_id=1008, ls_id={id:1}) [2025-08-15 10:15:14.910409] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.910421] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.910428] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.910436] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:14.910514] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.910526] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.910533] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.910540] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.910544] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3821, tenant_id=1008, ls_id={id:1}) [2025-08-15 10:15:14.910552] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:14.911634] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:14.932745] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.933904] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.935104] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.935135] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:14.935156] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.935170] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1001}, replicas:[{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:14.935284] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=88] build ls locality stat map(tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:15:14.935297] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.935310] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.935321] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.935329] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:14.935407] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.935422] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.935431] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.935440] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.935447] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3873, tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:15:14.935458] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:14.936601] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:14.957705] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.958880] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.960043] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.960065] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:14.960081] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.960091] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1002}, replicas:[{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:14.960193] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=80] build ls locality stat map(tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:15:14.960200] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.960209] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=7] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.960218] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.960226] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:14.960302] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.960314] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.960322] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.960329] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.960335] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3811, tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:15:14.960344] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:14.961414] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:14.982375] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.983553] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.984714] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:14.984739] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:14.984757] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.984769] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1003}, replicas:[{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:14.984873] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=82] build ls locality stat map(tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:15:14.984882] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.984897] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.984907] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:14.984919] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:14.985006] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=64] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.985021] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.985031] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.985040] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:14.985049] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3818, tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:15:14.985060] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:14.986141] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94544, tenant_id=1008, acc_dr_task=0) [2025-08-15 10:15:14.989335] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58C2-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.990013] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9A5-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:14.993239] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=6] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:15.101805] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58BC-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.134097] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF09D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.146576] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF03E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.161050] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F5-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.161064] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4D-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.163604] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F5-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:15.163619] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4D-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:15.163619] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F5-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.163635] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4D-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.183293] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CF5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.197863] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9AA-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.205793] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227715204613}) [2025-08-15 10:15:15.205810] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:15.205823] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:15:15.211670] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227715210522}) [2025-08-15 10:15:15.211686] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:15.211698] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:15:15.212862] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227715211710}) [2025-08-15 10:15:15.212876] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:15.212888] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:15:15.232399] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFD-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.232704] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.232784] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A7-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.233062] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7836-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.233113] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C8-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.235654] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFD-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.235777] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A7-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.235969] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02F-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.235989] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C8-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.236095] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7836-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.237730] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFD-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.237838] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFD-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.237916] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A7-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.237952] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C8-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.238049] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A7-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.238081] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C8-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.238097] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02F-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.238184] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC02F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.238306] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7836-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.238388] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7836-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.282613] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E55-0-0] [lt=9] start do ObArchiveSchedulerService round(round=101514) [2025-08-15 10:15:15.286172] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E55-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:15:15.286188] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E55-0-0] [lt=11][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1014 AND dest_no = 0 AND name = 'state', exec_tenant_id=1013) [2025-08-15 10:15:15.286237] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E55-0-0] [lt=9][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1014, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:15.286250] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E55-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1014, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:15:15.286260] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [709635][T1013_ArchiveSv][T1013][YB420ACB0122-000633A33EA44E55-0-0] [lt=7][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1014, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:15.296798] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7626-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.299907] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7626-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.301774] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7626-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.301872] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7626-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.351370] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0F-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.351418] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8830-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.353896] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0F-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:15.353914] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A0F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.353997] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8830-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:15.354017] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8830-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.439173] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6854-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.458456] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98CCB-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}) [2025-08-15 10:15:15.458497] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98CCB-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:15.459164] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:15.459185] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:15.459191] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:15.498879] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E70A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.498888] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF09B-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.529766] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A7-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:15.530885] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:15.530896] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:15.530901] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:15.531604] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:15.531615] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:15.531624] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:15.532311] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A7-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:15.537571] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:15.537583] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:15.537591] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:15.543192] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:15.543205] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:15.543210] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:15.548406] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:15.548418] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:15.548424] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:15.611438] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:15.612648] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224115611363, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:15:15.612671] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224120612669, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:15:15.631517] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9589-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.634113] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9589-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:15.634132] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9589-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.649098] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8855-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.651660] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8855-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:15.651676] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8855-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.653040] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C9-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:15.653256] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD523E-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:15.653978] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC62F-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.655576] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8C9-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:15.655667] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8357-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.655798] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD523E-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:15.657276] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438021][T1006_TimerWK0_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=7] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:15.658790] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8357-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.660695] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8357-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.660785] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8357-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:15.677903] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1C-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.680529] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1C-0-0] [lt=71] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:15.680547] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:15.687806] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6857-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.699109] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD69-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.705173] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:15.705187] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:15.705193] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:15.725800] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8674-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:15.728374] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8674-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:15.730255] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52DD-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:15.732756] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52DD-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:15.748409] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF042-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.748406] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD78-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.816720] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F83-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:15.819296] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F83-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:15.846094] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CF8-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.856080] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6287-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:15.858652] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6287-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:15.871391] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B352-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:15.900395] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [438062][T1006_STSWorker][T1006][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224115544878, tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224113790710004, replayable_scn:1755224113790710004, readable_scn:1755224113790710004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:15.934967] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:15.936152] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224115926158, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:15:15.936177] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224120936176, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:15:15.939441] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:15.940634] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224115939367, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:15:15.940654] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224120940653, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:15:15.940671] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224120940669, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:15.989413] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58C3-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:15.990092] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9A6-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.005257] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2845) [17347][TimerWK0_HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=12] need remove task(remove_task_ids=[]) [2025-08-15 10:15:16.005273] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2864) [17347][TimerWK0_HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=13] remove all timeout ddl task succeed [2025-08-15 10:15:16.101880] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58BD-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.115494] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [2619609][T1011_BackupDat][T1011][YB420ACB0122-0006338CC14639D0-0-0] [lt=15] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:16.134173] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF09E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.146650] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF03F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.166384] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4E-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.166384] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F6-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.168908] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4E-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:16.168955] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F6-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:16.168928] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4E-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.168973] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F6-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.178961] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:16.180170] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224116178884, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:15:16.180191] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224121180190, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:15:16.181207] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:16.182399] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224116181139, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:15:16.182418] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224121182417, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:16.183355] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CF6-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.197945] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9AB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.238815] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFE-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.238954] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C9-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.239020] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A8-0-0] [lt=11] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224115793896004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224115793896004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224115793896004, v:0}}], last_dump_ts_=1755224111207821, force_dump=false) [2025-08-15 10:15:16.239047] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A8-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.239191] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC030-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.239331] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7837-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.242030] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.242107] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFE-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.242186] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC030-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224115793896003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224115793896004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224115793896004, v:0}}], last_dump_ts_=1755224111210744, force_dump=false) [2025-08-15 10:15:16.242214] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC030-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.242237] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A8-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224115793896004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224115793896004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224115793896004, v:0}}], last_dump_ts_=1755224111210874, force_dump=false) [2025-08-15 10:15:16.242259] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A8-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.242315] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7837-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.244180] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFE-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.244187] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.244246] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC030-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.244289] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76C9-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.244313] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFE-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.244341] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC030-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.244402] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7837-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.244416] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A8-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.244489] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7837-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.244507] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A8-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.264271] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [1487706][T1003_BackupDat][T1003][YB420ACB0122-0006337FE347F6D6-0-0] [lt=13] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:16.269307] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D2C-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}) [2025-08-15 10:15:16.269351] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D2C-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:15:16.302921] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7627-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.305973] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7627-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.307896] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7627-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.308043] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7627-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.357908] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A10-0-0] [lt=38] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.357908] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8831-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.361761] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8831-0-0] [lt=98] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:16.361765] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A10-0-0] [lt=98] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:16.361782] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8831-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.361781] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A10-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.439250] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6855-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.458983] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:16.458994] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:16.458999] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:16.498957] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E70B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.498990] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF09C-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.503355] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF209-0-0] [lt=9] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.518684] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D57E4-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81054688GB}) [2025-08-15 10:15:16.518727] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D57E4-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=47, ddl_arg=NULL) [2025-08-15 10:15:16.530845] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224114970454004, replayable_scn:1755224114970454004, readable_scn:1755224114970454004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224116528260, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224114970454004, replayable_scn:1755224114970454004, readable_scn:1755224114970454004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224116528260, ora_rowscn_=1755224116305158000, content_changed=true) [2025-08-15 10:15:16.531043] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:16.531054] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:16.531063] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:16.531680] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:16.531692] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:16.531699] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:16.537120] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:16.537134] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:16.537141] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:16.537694] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A8-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:16.540237] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A8-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:16.543208] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:16.543219] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:16.543225] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:16.548483] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:16.548495] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:16.548501] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:16.621259] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [728059][T1017_BackupDat][T1017][YB420ACB0122-000633A355859844-0-0] [lt=15] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:16.636826] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958A-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.637252] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [728058][T1017_BACKUP_SC][T1017][YB420ACB0122-000633A35572C1CC-0-0] [lt=15] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:15:16.639327] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958A-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:16.639345] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.654414] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8856-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.657025] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8856-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:16.657043] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8856-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.660956] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8CA-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:16.661050] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD523F-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:16.661765] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8358-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.663504] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD523F-0-0] [lt=9] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:16.663509] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8CA-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:16.664835] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8358-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.666772] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8358-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.666860] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8358-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:16.683342] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1D-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.685976] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1D-0-0] [lt=74] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:16.685994] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:16.687884] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6858-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.699188] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD6A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.705183] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:16.705195] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:16.705201] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:16.733795] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8675-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:16.734987] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8675-0-0] [lt=13] log restore source is empty, just skip(ret=-4018, user_tenant_id=1006) [2025-08-15 10:15:16.737471] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8675-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:16.738024] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52DE-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:16.740520] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52DE-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:16.748477] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD79-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.748479] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF043-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.760238] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475908][T1008_TimerWK0_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:16.770144] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:16.820512] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:16.821758] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224116820437, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:15:16.821782] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224121821780, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:15:16.824753] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F84-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:16.827280] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F84-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:16.846165] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CF9-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.864059] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6288-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:16.866665] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6288-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:16.872775] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B353-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:16.989484] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58C4-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:16.990171] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9A7-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.095723] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727134][T1016_TimerWK2_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:17.101959] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58BE-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.134249] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF09F-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.146722] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF040-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.171686] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4F-0-0] [lt=42] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.171690] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F7-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.174305] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4F-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:17.174309] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F7-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:17.174323] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F7-0-0] [lt=10] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.174324] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF4F-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.183429] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CF7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.196016] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727959][T1018_TimerWK3_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:17.198020] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9AC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.245248] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CA-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.245346] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC031-0-0] [lt=9] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224116795321003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224116795321004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224116795321004, v:0}}], last_dump_ts_=1755224112213977, force_dump=false) [2025-08-15 10:15:17.245362] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFF-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.245372] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC031-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.245458] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A9-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.245705] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7838-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.248336] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFF-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.248373] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CA-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.248439] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A9-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.248458] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC031-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.249010] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7838-0-0] [lt=17] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224116795321003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224116795321003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224116795321003, v:0}}], last_dump_ts_=1755224112217084, force_dump=false) [2025-08-15 10:15:17.249046] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7838-0-0] [lt=27] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.250329] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=33] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:17.250363] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CA-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.250397] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.250418] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFF-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.250465] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC031-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.250478] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413A9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.250545] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9BFF-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.250562] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC031-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.250567] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CA-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.251534] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7838-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.251625] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7838-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.309027] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7628-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.310308] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487605][T1004_TimerWK2_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:17.311266] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=10] mview_mds_op(tx_count=0) [2025-08-15 10:15:17.311277] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=8] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=852) [2025-08-15 10:15:17.312083] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7628-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.314002] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7628-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.314129] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7628-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.325166] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:17.326407] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224117325095, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:15:17.326429] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224122326428, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:15:17.364510] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A11-0-0] [lt=47] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.364526] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8832-0-0] [lt=45] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.367083] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A11-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:17.367101] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A11-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.367134] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8832-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:17.367151] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8832-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.410933] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E5D-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224117410528000, start_id=1755224117410528000, end_id=1755224117410528001) [2025-08-15 10:15:17.411509] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:17.411605] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E5D-0-0] [lt=23] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224117410528000, v:0}, tenant_id=1017) [2025-08-15 10:15:17.412735] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224117411433, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:15:17.412756] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224122412755, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:15:17.439323] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6856-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.458879] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224115292385004, replayable_scn:1755224115292385004, readable_scn:1755224115292385004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224117455714, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224115292385004, replayable_scn:1755224115292385004, readable_scn:1755224115292385004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224117455714, ora_rowscn_=1755224116663867000, content_changed=true) [2025-08-15 10:15:17.458907] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A08C5-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}) [2025-08-15 10:15:17.458954] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A08C5-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=51, ddl_arg=NULL) [2025-08-15 10:15:17.459055] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:17.459068] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:17.459074] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:17.459081] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1014, cost_time_us=179, return_code_array=[0, 0, 0], broadcast_times=2) [2025-08-15 10:15:17.499032] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E70C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.499064] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF09D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.531104] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:17.531117] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:17.531123] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:17.531580] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224115793896004, replayable_scn:1755224115793896004, readable_scn:1755224115793896004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224117528325, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224115793896004, replayable_scn:1755224115793896004, readable_scn:1755224115793896004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224117528325, ora_rowscn_=1755224117247553000, content_changed=true) [2025-08-15 10:15:17.531747] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:17.531757] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:17.531765] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:17.533044] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:17.537027] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224115935935002, replayable_scn:1755224115935935002, readable_scn:1755224115935935002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224117534647, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224115935935002, replayable_scn:1755224115935935002, readable_scn:1755224115935935002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224117534647, ora_rowscn_=1755224117246500000, content_changed=true) [2025-08-15 10:15:17.537209] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:17.537220] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:17.537227] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:17.543126] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224115851779002, replayable_scn:1755224115851779002, readable_scn:1755224115851779002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224117539912, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224115851779002, replayable_scn:1755224115851779002, readable_scn:1755224115851779002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224117539912, ora_rowscn_=1755224117246500000, content_changed=true) [2025-08-15 10:15:17.543326] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:17.543349] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:17.543360] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:17.544775] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:17.545645] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A9-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:17.545653] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E66-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224117545178000, start_id=1755224117545178000, end_id=1755224117545178001) [2025-08-15 10:15:17.546002] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA668-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224117545178000, start_id=1755224117545178000, end_id=1755224117545178001) [2025-08-15 10:15:17.546338] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E66-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224117545178000, v:0}, tenant_id=1015) [2025-08-15 10:15:17.546711] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA668-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224117545178000, v:0}, tenant_id=1005) [2025-08-15 10:15:17.548296] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38A9-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:17.548479] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224115793896004, replayable_scn:1755224115793896004, readable_scn:1755224115793896004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224117545178, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224115793896004, replayable_scn:1755224115793896004, readable_scn:1755224115793896004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224117545178, ora_rowscn_=1755224117247553000, content_changed=true) [2025-08-15 10:15:17.548655] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:17.548665] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:17.548675] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:17.642069] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958B-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.644583] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958B-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:17.644601] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958B-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.659061] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC630-0-0] [lt=13] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.659773] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8857-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.662358] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8857-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:17.662373] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8857-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.667810] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8359-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.668744] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5240-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:17.668802] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8CB-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:17.670884] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8359-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.671288] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5240-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:17.672486] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8CB-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:17.672852] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8359-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.672936] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8359-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:17.687958] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6859-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.688755] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.691288] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1E-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:17.691305] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1E-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:17.699262] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD6B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.705071] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224115793896003, replayable_scn:1755224115793896003, readable_scn:1755224115793896003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224117702015, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224115793896003, replayable_scn:1755224115793896003, readable_scn:1755224115793896003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224117702015, ora_rowscn_=1755224117247553000, content_changed=true) [2025-08-15 10:15:17.705264] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:17.705276] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:17.705281] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:17.742861] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8676-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:17.745406] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8676-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:17.745801] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52DF-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:17.748362] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52DF-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:17.748552] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF044-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.748551] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD7A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.775283] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36C6-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224117774602000, start_id=1755224117774602000, end_id=1755224117774602001) [2025-08-15 10:15:17.775977] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36C6-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224117774602000, v:0}, tenant_id=1011) [2025-08-15 10:15:17.789872] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=6] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:17.791116] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224117789795, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:15:17.791139] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224122791138, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:15:17.791581] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:17.792781] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224117791513, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:15:17.792801] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224122792800, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:15:17.832586] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F85-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:17.835115] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F85-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:17.846236] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CFA-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.872145] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6289-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:17.874154] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B354-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:17.874740] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6289-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:17.893375] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FAE-0-0] [lt=95] get number(ret=0, service_type_=0, range=1, base_id=1755224117892413000, start_id=1755224117892413000, end_id=1755224117892413001) [2025-08-15 10:15:17.894063] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FAE-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224117892413000, v:0}, tenant_id=1013) [2025-08-15 10:15:17.902252] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826D-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1005, thread_idx=0) [2025-08-15 10:15:17.903488] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1005, only_for_display=false) [2025-08-15 10:15:17.904703] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=17] start try ls disaster recovery(ls_status_info={tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:17.918591] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:17.919752] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:17.920904] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:17.920931] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1005, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:17.920953] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=15] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.920966] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1005, ls_id:{id:1}, replicas:[{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:17.921076] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=85] build ls locality stat map(tenant_id=1005, ls_id={id:1}) [2025-08-15 10:15:17.921084] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:17.921099] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:17.921108] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:17.921121] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:17.921200] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.921215] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.921225] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.921231] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.921240] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3832, tenant_id=1005, ls_id={id:1}) [2025-08-15 10:15:17.921247] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:17.922396] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18908, tenant_id=1005, acc_dr_task=0) [2025-08-15 10:15:17.923640] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1006, only_for_display=false) [2025-08-15 10:15:17.924852] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:17.938831] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:17.939991] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:17.941149] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:17.941170] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:17.941185] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.941194] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1}, replicas:[{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:17.941293] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=78] build ls locality stat map(tenant_id=1006, ls_id={id:1}) [2025-08-15 10:15:17.941301] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:17.941312] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:17.941321] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:17.941330] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:17.941412] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.941424] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.941432] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.941439] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.941446] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3775, tenant_id=1006, ls_id={id:1}) [2025-08-15 10:15:17.941452] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:17.942519] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:17.953325] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:17.954523] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224117953237, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:15:17.954548] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224122954547, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:15:17.964655] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:17.965838] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:17.966984] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:17.967007] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:17.967023] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.967035] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1001}, replicas:[{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:17.967149] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=89] build ls locality stat map(tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:15:17.967157] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:17.967171] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:17.967182] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:17.967193] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:17.967272] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.967286] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.967296] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.967301] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.967305] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3806, tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:15:17.967316] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:17.968471] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:17.989210] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:17.989557] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58C5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.990244] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9A8-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:17.990350] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:17.991497] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:17.991522] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:17.991541] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.991552] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1002}, replicas:[{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:17.991657] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=82] build ls locality stat map(tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:15:17.991668] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:17.991681] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:17.991693] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:17.991709] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:17.991787] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.991801] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.991811] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.991817] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:17.991826] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3753, tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:15:17.991836] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:17.993046] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:18.013607] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.014758] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.015918] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.015947] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:18.015966] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.015973] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=5] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1003}, replicas:[{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:18.016083] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=85] build ls locality stat map(tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:15:18.016091] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.016104] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.016116] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.016128] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:18.016204] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.016220] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.016231] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.016237] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.016246] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3766, tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:15:18.016252] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:18.017373] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94962, tenant_id=1006, acc_dr_task=0) [2025-08-15 10:15:18.039093] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:18.040314] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224118039022, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:15:18.040340] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224123040339, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:15:18.069880] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36AE-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224118069050000, start_id=1755224118069050000, end_id=1755224118069050001) [2025-08-15 10:15:18.070800] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36AE-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224118069050000, v:0}, tenant_id=1012) [2025-08-15 10:15:18.091052] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463B6-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224118090051000, start_id=1755224118090051000, end_id=1755224118090051001) [2025-08-15 10:15:18.091842] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463B6-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224118090051000, v:0}, tenant_id=1) [2025-08-15 10:15:18.102038] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58BF-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.134328] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0A0-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.146791] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF041-0-0] [lt=8] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.177049] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F8-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.177049] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF50-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.179555] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF50-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:18.179573] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF50-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.179616] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F8-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:18.179635] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F8-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.183466] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CF8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.198097] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9AD-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.217178] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:18.217196] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:18.223110] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:18.224326] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17350][TimerWK3_IngressService][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:18.251476] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AA-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.251619] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC032-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.251635] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C00-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.251805] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CB-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224117796703004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224117796703004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224117796703004, v:0}}], last_dump_ts_=1755224113220197, force_dump=false) [2025-08-15 10:15:18.251830] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CB-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.252587] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7839-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.254757] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C00-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.254815] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AA-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.254818] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC032-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.254948] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CB-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224117796703004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224117796703004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224117796703004, v:0}}], last_dump_ts_=1755224113223326, force_dump=false) [2025-08-15 10:15:18.254971] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CB-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.256350] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7839-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.256855] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C00-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.256915] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AA-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.256936] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC032-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.256962] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C00-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.257047] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CB-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.257061] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC032-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.257092] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AA-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.257188] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CB-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.258338] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7839-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.258496] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7839-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.269836] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B6334-0-0] [lt=12] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}) [2025-08-15 10:15:18.269879] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B6334-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=47, ddl_arg=NULL) [2025-08-15 10:15:18.315111] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7629-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.318228] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7629-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.320157] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7629-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.320237] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7629-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.369814] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A12-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.369929] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8833-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.372348] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A12-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:18.372364] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A12-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.372585] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8833-0-0] [lt=83] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:18.372600] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8833-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.439400] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6857-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.459214] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:18.459225] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:18.459231] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:18.499104] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E70D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.499137] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF09E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.503643] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF20A-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.519201] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBA6C-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81054688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81445312GB}) [2025-08-15 10:15:18.519243] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBA6C-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:18.531172] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:18.531184] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:18.531189] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:18.531783] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:18.531795] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:18.531801] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:18.537223] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:18.537236] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:18.537242] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:18.538511] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:18.543300] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:18.543311] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:18.543320] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:18.548639] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:18.548650] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:18.548656] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:18.551455] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEC-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1, thread_idx=0) [2025-08-15 10:15:18.552671] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-08-15 10:15:18.553569] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AA-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:18.553920] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:18.556105] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AA-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:18.568216] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.569409] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.570594] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.570614] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:18.570628] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.570638] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:18.570737] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=78] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-08-15 10:15:18.570745] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.570755] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.570764] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.570772] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:18.570847] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.570859] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.570867] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.570874] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.570880] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3810, tenant_id=1, ls_id={id:1}) [2025-08-15 10:15:18.570888] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:18.571968] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19297, tenant_id=1, acc_dr_task=0) [2025-08-15 10:15:18.597061] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:18.598295] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224118596992, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:15:18.598318] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224123598316, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:15:18.647323] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958C-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.649846] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958C-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:18.649864] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958C-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.665053] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8858-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.667630] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8858-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:18.667648] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8858-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.670777] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [2619733][T1011_MergeSche][T1011][YB420ACB0122-0006338CC1A2D970-0-0] [lt=68] zone_merge_mgr is already loaded(tenant_id=1011, global_merge_info={tenant_id:1011, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402005497, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194694858130, need_update:false}}, zone_merge_infos=[{tenant_id:1011, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194694860625, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402018052, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1011, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194694864262, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402054910, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1011, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194694866113, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402057588, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401984392000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:15:18.673931] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.676657] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5241-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:18.676895] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835A-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224118297449003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224118297449003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224118297449003, v:0}}], last_dump_ts_=1755224113646153, force_dump=false) [2025-08-15 10:15:18.676919] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835A-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.677804] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8CC-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:18.678950] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.679088] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:18.679177] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5241-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:18.680351] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8CC-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:18.688039] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C685A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.694065] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1F-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.696592] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1F-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:18.696613] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB1F-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:18.699340] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD6C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.705308] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:18.705329] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:18.705335] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:18.728347] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E56-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224118727495000, start_id=1755224118727495000, end_id=1755224118727495001) [2025-08-15 10:15:18.728991] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E56-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224118727495000, v:0}, tenant_id=1016) [2025-08-15 10:15:18.748621] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD7B-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.748620] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF045-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.750688] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8677-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:18.753177] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8677-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:18.753589] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E0-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:18.756153] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E0-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:18.840453] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F86-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:18.842936] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F86-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:18.846310] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CFB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.872368] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E5F-0-0] [lt=8] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1003, thread_idx=0) [2025-08-15 10:15:18.873630] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1003, only_for_display=false) [2025-08-15 10:15:18.874846] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:18.875534] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B355-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:18.880224] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB628A-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:18.882794] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB628A-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:18.888542] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.889695] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.890865] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.890892] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1003, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:18.890910] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.890922] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1003, ls_id:{id:1}, replicas:[{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:18.891056] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=105] build ls locality stat map(tenant_id=1003, ls_id={id:1}) [2025-08-15 10:15:18.891065] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.891078] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.891089] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.891100] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:18.891177] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.891190] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.891199] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.891204] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.891213] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3806, tenant_id=1003, ls_id={id:1}) [2025-08-15 10:15:18.891224] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:18.892297] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18667, tenant_id=1003, acc_dr_task=0) [2025-08-15 10:15:18.893517] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1004, only_for_display=false) [2025-08-15 10:15:18.894724] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:18.901945] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=17] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224118535056, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224116795321003, replayable_scn:1755224116795321003, readable_scn:1755224116795321003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:18.908410] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.909566] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.910709] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.910732] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:18.910746] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.910757] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:18.910861] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=81] build ls locality stat map(tenant_id=1004, ls_id={id:1}) [2025-08-15 10:15:18.910869] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.910882] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.910893] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.910902] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:18.910983] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.910997] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.911006] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.911011] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.911020] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3757, tenant_id=1004, ls_id={id:1}) [2025-08-15 10:15:18.911029] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:18.912107] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:18.933044] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.934230] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.935382] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.935405] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:18.935423] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.935435] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:18.935543] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=84] build ls locality stat map(tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:15:18.935551] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.935564] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.935575] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.935586] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:18.935663] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.935677] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.935687] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.935692] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.935701] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3826, tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:15:18.935711] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:18.936778] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:18.957615] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.958768] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.959931] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.959960] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:18.959978] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.959988] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1002}, replicas:[{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:18.960096] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=85] build ls locality stat map(tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:15:18.960104] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.960117] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.960128] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.960139] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:18.960214] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.960228] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.960238] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.960243] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.960252] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3789, tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:15:18.960258] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:18.961337] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:18.982155] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.983309] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.984476] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:18.984497] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:18.984510] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.984522] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1003}, replicas:[{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:18.984628] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=82] build ls locality stat map(tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:15:18.984636] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.984649] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.984660] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:18.984671] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:18.984749] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.984763] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.984772] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.984782] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:18.984789] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3812, tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:15:18.984799] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:18.985868] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93556, tenant_id=1004, acc_dr_task=0) [2025-08-15 10:15:18.989633] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58C6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:18.990316] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9A9-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.041033] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [2619737][T1012_MergeSche][T1012][YB420ACB0122-0006338CC242D8F0-0-0] [lt=16] zone_merge_mgr is already loaded(tenant_id=1012, global_merge_info={tenant_id:1012, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194400967887, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194695172923, need_update:false}}, zone_merge_infos=[{tenant_id:1012, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194695173510, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194400994787, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1012, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194695175345, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401001136, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1012, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194695177103, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401004882, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194400813511000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:15:19.102110] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58C0-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.134385] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0A1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.141348] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA649-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224119140812000, start_id=1755224119140812000, end_id=1755224119140812001) [2025-08-15 10:15:19.142133] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA649-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224119140812000, v:0}, tenant_id=1006) [2025-08-15 10:15:19.146866] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF042-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.182409] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF51-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.182440] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F9-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.183537] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CF9-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.184966] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF51-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:19.184981] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF51-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.185035] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F9-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:19.185050] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99F9-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.198171] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9AE-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.198794] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B8-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1017, thread_idx=0) [2025-08-15 10:15:19.200067] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1017, only_for_display=false) [2025-08-15 10:15:19.201323] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:19.206225] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A2-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224119206016000, start_id=1755224119206016000, end_id=1755224119206016001) [2025-08-15 10:15:19.207089] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A2-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224119206016000, v:0}, tenant_id=1008) [2025-08-15 10:15:19.215296] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.216441] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.217618] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.217640] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1017, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:19.217658] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.217670] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1017, ls_id:{id:1}, replicas:[{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:19.217772] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=81] build ls locality stat map(tenant_id=1017, ls_id={id:1}) [2025-08-15 10:15:19.217780] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.217794] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.217804] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.217816] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:19.217893] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.217908] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.217917] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.217921] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.217930] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3807, tenant_id=1017, ls_id={id:1}) [2025-08-15 10:15:19.217947] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=13] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:19.219030] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18963, tenant_id=1017, acc_dr_task=0) [2025-08-15 10:15:19.220249] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1018, only_for_display=false) [2025-08-15 10:15:19.221467] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:19.235437] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.236959] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.238327] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.238350] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:19.238368] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.238380] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1}, replicas:[{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:19.238489] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=86] build ls locality stat map(tenant_id=1018, ls_id={id:1}) [2025-08-15 10:15:19.238500] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.238516] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.238529] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=10] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.238541] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:19.238621] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.238638] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.238648] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.238658] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.238664] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=4404, tenant_id=1018, ls_id={id:1}) [2025-08-15 10:15:19.238674] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:19.240037] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:19.257896] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C01-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224118797983003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224119237575000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224119237575000, v:0}}], last_dump_ts_=1755224114226143, force_dump=false) [2025-08-15 10:15:19.257934] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C01-0-0] [lt=28] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.258067] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AB-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.258084] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC033-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.258126] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CC-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.259421] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783A-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224118797983003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224118797983003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224118797983003, v:0}}], last_dump_ts_=1755224114226508, force_dump=false) [2025-08-15 10:15:19.259444] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783A-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.260946] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C01-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224118797983003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224119207069000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224119207069000, v:0}}], last_dump_ts_=1755224114229283, force_dump=false) [2025-08-15 10:15:19.260970] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C01-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.261040] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AB-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.261161] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CC-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.261283] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC033-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.262241] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.262413] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783A-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.263032] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C01-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.263113] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AB-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.263162] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C01-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.263174] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CC-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.263199] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AB-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.263244] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC033-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.263277] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CC-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.263338] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC033-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.263427] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.264310] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.264403] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783A-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.264615] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.264642] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:19.264659] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.264669] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1001}, replicas:[{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:19.264773] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:15:19.264788] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.264801] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.264812] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.264824] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:19.264904] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.264920] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.264931] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.264943] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.264952] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3916, tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:15:19.264959] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:19.266063] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:19.286945] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.288100] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.289293] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.289315] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:19.289334] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.289346] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1002}, replicas:[{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:19.289450] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=81] build ls locality stat map(tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:15:19.289459] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.289473] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.289479] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.289490] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:19.289582] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=66] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.289596] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.289605] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.289611] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.289618] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3825, tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:15:19.289629] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:19.290754] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:19.311709] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.312870] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.314043] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:19.314066] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:19.314083] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.314094] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1003}, replicas:[{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:19.314197] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=81] build ls locality stat map(tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:15:19.314205] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.314218] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.314228] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:19.314235] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:19.314313] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.314327] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.314337] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.314342] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:19.314352] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3786, tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:15:19.314361] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:19.315440] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=96394, tenant_id=1018, acc_dr_task=0) [2025-08-15 10:15:19.321221] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762A-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224118964100002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224118964100002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224118964100002, v:0}}], last_dump_ts_=1755224114290584, force_dump=false) [2025-08-15 10:15:19.321246] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762A-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.324328] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762A-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224118964100002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224118964100002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224118964100002, v:0}}], last_dump_ts_=1755224114293620, force_dump=false) [2025-08-15 10:15:19.324356] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762A-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.326605] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.326702] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.375029] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A13-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.375330] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8834-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.377567] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A13-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:19.377585] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A13-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.377954] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8834-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:19.377975] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8834-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.439449] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6858-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.459122] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:19.459133] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:19.459137] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:19.459377] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A08C6-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}) [2025-08-15 10:15:19.459421] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A08C6-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:15:19.460451] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:19.499180] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E70E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.499211] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF09F-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.531096] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224117965042002, replayable_scn:1755224117965042002, readable_scn:1755224117965042002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224119527921, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224117965042002, replayable_scn:1755224117965042002, readable_scn:1755224117965042002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224119527921, ora_rowscn_=1755224119322719000, content_changed=true) [2025-08-15 10:15:19.531288] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:19.531298] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:19.531306] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:19.531826] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:19.531837] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:19.531845] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:19.532587] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:19.537254] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:19.537264] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:19.537269] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:19.543345] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:19.543358] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:19.543364] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:19.548683] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:19.548704] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:19.548711] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:19.550094] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:19.561395] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AB-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:19.563888] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AB-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:19.617871] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:19.619109] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224119617798, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:15:19.619133] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224124619131, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:15:19.652535] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958D-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.655061] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958D-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:19.655076] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958D-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.663958] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC631-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.670361] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8859-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.672928] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8859-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:19.672954] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8859-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.680064] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835B-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224119298571004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224119298571004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224119298571004, v:0}}], last_dump_ts_=1755224114649508, force_dump=false) [2025-08-15 10:15:19.680090] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835B-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.683093] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835B-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.685031] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.685118] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835B-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:19.685737] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5242-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:19.685785] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8CD-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:19.688118] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C685B-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.688330] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8CD-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:19.688678] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5242-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:19.699313] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB20-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.699418] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD6D-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.701850] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB20-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:19.701870] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB20-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:19.702884] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E2-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224119702260000, start_id=1755224119702260000, end_id=1755224119702260001) [2025-08-15 10:15:19.703614] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E2-0-0] [lt=13] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224119702260000, v:0}, tenant_id=1003) [2025-08-15 10:15:19.703634] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E2-0-0] [lt=12] zone_merge_mgr is already loaded(tenant_id=1003, global_merge_info={tenant_id:1003, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403772779, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194756784394, need_update:false}}, zone_merge_infos=[{tenant_id:1003, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194756785330, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403861743, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1003, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194756787126, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403865394, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1003, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194756788906, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403869575, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194403729032000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:15:19.705398] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:19.705409] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:19.705416] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:19.706723] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:19.748692] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD7C-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.748694] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF046-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.758466] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8678-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:19.761089] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8678-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:19.761389] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E1-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:19.763917] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E1-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:19.842292] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5BF-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224119841956000, start_id=1755224119841956000, end_id=1755224119841956001) [2025-08-15 10:15:19.843097] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5BF-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224119841956000, v:0}, tenant_id=1004) [2025-08-15 10:15:19.846386] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CFC-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.848305] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F87-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:19.850860] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F87-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:19.876960] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B356-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:19.888073] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB628B-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:19.890575] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB628B-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:19.989705] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58C7-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.990390] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9AA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:19.994537] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:19.996521] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=11] mview_mds_op(tx_count=0) [2025-08-15 10:15:19.996533] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=8] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=843) [2025-08-15 10:15:20.102184] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58C1-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.134457] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0A2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.146913] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF043-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.174357] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C0-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224120174124000, start_id=1755224120174124000, end_id=1755224120174124001) [2025-08-15 10:15:20.175040] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C0-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224120174124000, v:0}, tenant_id=1007) [2025-08-15 10:15:20.183607] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CFA-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.187720] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF52-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.187747] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FA-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.190287] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF52-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:20.190308] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF52-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.190344] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FA-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:20.190366] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FA-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.198243] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9AF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.264201] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AC-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.264225] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C02-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.264281] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CD-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.264302] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC034-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.265367] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.267585] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AC-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.267580] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C02-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.267599] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC034-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.267660] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CD-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.268501] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783B-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.269633] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AC-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.269686] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC034-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.269698] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C02-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.269753] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AC-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.269750] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CD-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.269795] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C02-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.269861] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CD-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.269878] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC034-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.270339] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BB5-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}) [2025-08-15 10:15:20.270377] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BB5-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:15:20.270971] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.271091] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.327719] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762B-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.330869] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762B-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.332827] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.332915] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.380296] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A14-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.380658] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8835-0-0] [lt=41] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.382875] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A14-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:20.382893] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A14-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.383213] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8835-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:20.383232] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8835-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.439524] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6859-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.459026] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224118296398004, replayable_scn:1755224118296398004, readable_scn:1755224118296398004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224120455918, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224118296398004, replayable_scn:1755224118296398004, readable_scn:1755224118296398004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224120455918, ora_rowscn_=1755224119682326000, content_changed=true) [2025-08-15 10:15:20.459209] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:20.459221] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:20.459227] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:20.459237] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1014, cost_time_us=188, return_code_array=[0, 0, 0], broadcast_times=2) [2025-08-15 10:15:20.499262] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E70F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.499283] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0A0-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.503913] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF20B-0-0] [lt=10] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.519749] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D57F0-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81445312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81835938GB}) [2025-08-15 10:15:20.519788] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D57F0-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:15:20.531200] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:20.531213] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:20.531218] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:20.531702] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224118797983004, replayable_scn:1755224118797983004, readable_scn:1755224118797983004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224120528424, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224118797983004, replayable_scn:1755224118797983004, readable_scn:1755224118797983004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224120528424, ora_rowscn_=1755224120266658000, content_changed=true) [2025-08-15 10:15:20.531862] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:20.531873] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:20.531878] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:20.537760] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224118799037004, replayable_scn:1755224118799037004, readable_scn:1755224118799037004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224120534746, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224118799037004, replayable_scn:1755224118799037004, readable_scn:1755224118799037004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224120534746, ora_rowscn_=1755224120266658000, content_changed=true) [2025-08-15 10:15:20.537923] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:20.537934] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:20.537945] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:20.543224] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224118975692003, replayable_scn:1755224118975692003, readable_scn:1755224118975692003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224120539993, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224118975692003, replayable_scn:1755224118975692003, readable_scn:1755224118975692003, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224120539993, ora_rowscn_=1755224120266658000, content_changed=true) [2025-08-15 10:15:20.543395] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:20.543406] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:20.543413] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:20.548486] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224119141866000, replayable_scn:1755224119141866000, readable_scn:1755224119141866000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224120545260, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224119141866000, replayable_scn:1755224119141866000, readable_scn:1755224119141866000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224120545260, ora_rowscn_=1755224120266658000, content_changed=true) [2025-08-15 10:15:20.548632] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:20.548641] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:20.548648] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:20.569586] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AC-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:20.570972] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AC-0-0] [lt=13] log restore source is empty, just skip(ret=-4018, user_tenant_id=1004) [2025-08-15 10:15:20.573422] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AC-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:20.612739] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:20.613955] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224120612669, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:15:20.613978] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224125613976, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:15:20.657810] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958E-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.660325] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958E-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:20.660356] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958E-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.663539] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [3413782][T1006_TimerWK4_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=17] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:20.675700] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885A-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.678258] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885A-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:20.678274] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885A-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.686142] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.688199] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C685C-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.689104] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835C-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.690999] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.691173] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:20.693796] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8CE-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:20.694083] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5243-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:20.696339] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8CE-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:20.696575] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5243-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:20.699496] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD6E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.704614] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB21-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.705301] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224118797983003, replayable_scn:1755224118797983003, readable_scn:1755224118797983003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224120701871, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224118797983003, replayable_scn:1755224118797983003, readable_scn:1755224118797983003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224120701871, ora_rowscn_=1755224120267712000, content_changed=true) [2025-08-15 10:15:20.705485] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:20.705497] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:20.705507] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:20.707141] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB21-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:20.707160] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB21-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:20.748762] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD7D-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.748768] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF047-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.766531] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8679-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:20.769255] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E2-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:20.769371] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8679-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:20.771754] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E2-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:20.846446] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CFD-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.856215] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F88-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:20.858761] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F88-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:20.878316] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B357-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:20.895951] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB628C-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:20.898476] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB628C-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:20.937421] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:20.938626] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224120936176, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:15:20.938652] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224125938650, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:15:20.944936] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:20.946139] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224120940653, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:15:20.946160] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224125946159, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:15:20.946176] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224125946174, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:20.989781] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58C8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:20.990467] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9AB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.049476] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B0E-0-0] [lt=12] [RESTORE] restore scheduler quit [2025-08-15 10:15:21.050730] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B0E-0-0] [lt=12] get targets(ret=0, jobs=[], exec_tenant_id=1005, sql=select * from __all_recover_table_job) [2025-08-15 10:15:21.051928] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B0F-0-0] [lt=15] get targets(ret=0, jobs=[], exec_tenant_id=1005, sql=select * from __all_import_table_job) [2025-08-15 10:15:21.051958] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B10-0-0] [lt=20][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:21.052041] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B11-0-0] [lt=11] [RESTORE] restore scheduler start [2025-08-15 10:15:21.052050] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B12-0-0] [lt=7] [RESTORE] try process restore job [2025-08-15 10:15:21.053175] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [438121][T1005_REST_SER][T1005][YB420ACB0122-0006338206FB5B12-0-0] [lt=3] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:15:21.102229] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58C2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.134530] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0A3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.146989] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF044-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.154318] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68CF-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1015, thread_idx=0) [2025-08-15 10:15:21.155585] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1015, only_for_display=false) [2025-08-15 10:15:21.156803] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:21.170561] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.171716] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.172856] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.172878] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1015, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:21.172894] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.172905] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1015, ls_id:{id:1}, replicas:[{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:21.173014] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=85] build ls locality stat map(tenant_id=1015, ls_id={id:1}) [2025-08-15 10:15:21.173022] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.173035] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.173045] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.173056] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:21.173135] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.173145] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.173154] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.173159] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.173168] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3776, tenant_id=1015, ls_id={id:1}) [2025-08-15 10:15:21.173174] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:21.174235] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18651, tenant_id=1015, acc_dr_task=0) [2025-08-15 10:15:21.175451] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1016, only_for_display=false) [2025-08-15 10:15:21.176650] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:21.180271] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:21.181494] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224121180190, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:15:21.181516] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224126181515, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:15:21.182483] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:21.183679] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224121182417, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:15:21.183683] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CFB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.183699] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224126183698, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:21.190386] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.191533] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.192670] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.192692] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:21.192710] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.192721] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1}, replicas:[{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:21.192824] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=80] build ls locality stat map(tenant_id=1016, ls_id={id:1}) [2025-08-15 10:15:21.192833] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.192846] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.192857] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.192866] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:21.192949] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.192966] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.192975] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.192986] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.192991] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3753, tenant_id=1016, ls_id={id:1}) [2025-08-15 10:15:21.193001] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:21.193043] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF53-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.193068] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FB-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.194126] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:21.195669] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF53-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:21.195671] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FB-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:21.195688] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF53-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.195687] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FB-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.198317] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9B0-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.214776] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.215932] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.217072] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.217094] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:21.217106] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.217117] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1001}, replicas:[{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:21.217228] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=86] build ls locality stat map(tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:15:21.217235] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.217247] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.217257] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.217270] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:21.217352] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.217368] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.217378] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.217383] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.217388] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3741, tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:15:21.217398] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:21.218492] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:21.228681] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=16][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:21.228699] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:21.234652] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=1][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:21.235833] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=1][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:21.239738] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.240879] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.242015] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.242036] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:21.242048] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.242057] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1002}, replicas:[{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:21.242156] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=77] build ls locality stat map(tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:15:21.242165] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.242176] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.242184] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.242193] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:21.242269] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.242277] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=3] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.242285] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.242291] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.242298] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3684, tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:15:21.242306] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:21.243382] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:21.263921] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.265059] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.266198] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:21.266218] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:21.266233] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.266241] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1003}, replicas:[{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:21.266342] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=78] build ls locality stat map(tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:15:21.266349] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.266360] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.266369] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:21.266377] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:21.266451] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=54] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.266462] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.266469] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.266474] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:21.266481] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3681, tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:15:21.266488] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:21.267629] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93381, tenant_id=1016, acc_dr_task=0) [2025-08-15 10:15:21.270714] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C03-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.270735] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AD-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224120800670004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224120800670004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224120800670004, v:0}}], last_dump_ts_=1755224116239019, force_dump=false) [2025-08-15 10:15:21.270763] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AD-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.270777] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CE-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.270819] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC035-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.271991] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.273818] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CE-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.273853] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AD-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224120800670004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224120800670004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224120800670004, v:0}}], last_dump_ts_=1755224116242236, force_dump=false) [2025-08-15 10:15:21.273878] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AD-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.273881] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C03-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.273976] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC035-0-0] [lt=9] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224120800670003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224120800670004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224120800670004, v:0}}], last_dump_ts_=1755224116242185, force_dump=false) [2025-08-15 10:15:21.273998] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC035-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.274889] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783C-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.275764] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CE-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.275806] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AD-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.275807] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C03-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.275847] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CE-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.275885] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AD-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.275884] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C03-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.276058] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC035-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.276141] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC035-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.276872] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.277058] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.333908] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.336860] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762C-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.338807] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.338931] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.384751] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E4D-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224121384450000, start_id=1755224121384450000, end_id=1755224121384450001) [2025-08-15 10:15:21.385545] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E4D-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224121384450000, v:0}, tenant_id=1018) [2025-08-15 10:15:21.385656] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A15-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.385955] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8836-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.388197] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A15-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:21.388215] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A15-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.388523] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8836-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:21.388541] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8836-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.439597] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C685A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.459274] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:21.459285] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:21.459290] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:21.459851] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98CDF-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}) [2025-08-15 10:15:21.459892] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98CDF-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:21.499319] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0A1-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.499337] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E710-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.531312] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:21.531331] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:21.531338] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:21.531956] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:21.531968] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:21.531976] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:21.537971] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:21.537982] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:21.537988] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:21.543415] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:21.543431] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:21.543439] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:21.548719] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:21.548733] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:21.548738] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:21.578695] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AD-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:21.581171] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AD-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:21.663034] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958F-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.665511] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958F-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:21.665527] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D958F-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.668849] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC632-0-0] [lt=13] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.680973] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885B-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.683540] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885B-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:21.683558] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.688276] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C685D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.692322] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.695433] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835D-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.697366] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.697485] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:21.699571] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD6F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.701645] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8CF-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:21.701807] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5244-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:21.704174] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8CF-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:21.705012] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5244-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:21.706178] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:21.706191] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:21.706196] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:21.709964] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB22-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.712559] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB22-0-0] [lt=70] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:21.712576] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB22-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:21.748834] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD7E-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.748842] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF048-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.765613] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:21.771487] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709535][T1014_TimerWK2_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=6] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:21.774697] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB867A-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:21.776955] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E3-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:21.777203] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB867A-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:21.779421] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E3-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:21.779712] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7F9D-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224121779695000, start_id=1755224121779695000, end_id=1755224121779695001) [2025-08-15 10:15:21.780521] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7F9D-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224121779695000, v:0}, tenant_id=1014) [2025-08-15 10:15:21.821859] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:21.823079] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224121821780, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:15:21.823103] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224126823101, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:15:21.846523] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CFE-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.864118] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F89-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:21.866614] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F89-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:21.879679] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B358-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:21.903511] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=11] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224121534823, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224120242510000, replayable_scn:1755224120242510000, readable_scn:1755224120242510000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:21.903737] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB628D-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:21.906221] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB628D-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:21.989853] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58C9-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:21.990545] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9AC-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.101124] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=34] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:22.102308] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58C3-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.134604] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0A4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.147063] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF045-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.183756] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CFC-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.197359] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:22.198393] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9B1-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.198385] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FC-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.198393] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF54-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.200936] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF54-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:22.200949] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FC-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:22.200960] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF54-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.200967] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FC-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.240946] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E6-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1013, thread_idx=0) [2025-08-15 10:15:22.242189] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1013, only_for_display=false) [2025-08-15 10:15:22.243437] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:22.251665] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619488][T1012_TimerWK3_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:22.258098] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.259295] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.260484] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.260508] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1013, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:22.260525] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.260537] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1013, ls_id:{id:1}, replicas:[{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:22.260641] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=81] build ls locality stat map(tenant_id=1013, ls_id={id:1}) [2025-08-15 10:15:22.260649] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.260662] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.260672] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.260683] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:22.260761] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.260774] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.260784] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.260789] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.260794] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3878, tenant_id=1013, ls_id={id:1}) [2025-08-15 10:15:22.260805] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:22.261899] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19710, tenant_id=1013, acc_dr_task=0) [2025-08-15 10:15:22.263178] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1014, only_for_display=false) [2025-08-15 10:15:22.264418] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:22.270862] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9F9E9-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}) [2025-08-15 10:15:22.270903] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9F9E9-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:15:22.276896] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CF-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.276908] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AE-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.276927] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C04-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.277082] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC036-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224121801772004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224121801772006, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224121801772006, v:0}}], last_dump_ts_=1755224117245346, force_dump=false) [2025-08-15 10:15:22.277105] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC036-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.277948] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.278521] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.279737] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.279989] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CF-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.279999] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AE-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.280187] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C04-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.280257] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC036-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.280878] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783D-0-0] [lt=19] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224121801772004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224121801772005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224121801772005, v:0}}], last_dump_ts_=1755224117249009, force_dump=false) [2025-08-15 10:15:22.280905] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783D-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.280918] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.280951] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:22.280973] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=13] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.280987] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1}, replicas:[{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:22.281121] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=105] build ls locality stat map(tenant_id=1014, ls_id={id:1}) [2025-08-15 10:15:22.281131] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.281143] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.281155] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.281167] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:22.281246] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.281256] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.281265] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.281271] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.281280] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3928, tenant_id=1014, ls_id={id:1}) [2025-08-15 10:15:22.281290] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:22.281913] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CF-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.282040] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76CF-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.282043] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AE-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.282166] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AE-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.282241] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C04-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.282323] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC036-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.282332] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C04-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.282406] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC036-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.282458] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:22.282803] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783D-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.282881] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.303719] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.304920] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.306105] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.306127] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:22.306145] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.306155] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1001}, replicas:[{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:22.306255] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=78] build ls locality stat map(tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:15:22.306265] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.306277] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.306285] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.306294] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:22.306370] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.306381] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.306389] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.306395] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.306401] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3887, tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:15:22.306409] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:22.307484] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:22.311646] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=20] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:22.326466] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:22.327710] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224122326428, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:15:22.327734] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224127327732, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:15:22.328784] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.329961] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.331139] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.331163] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:22.331178] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.331188] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1002}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:22.331292] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=82] build ls locality stat map(tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:15:22.331300] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.331311] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.331319] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.331326] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:22.331402] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.331413] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.331420] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.331427] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.331434] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3867, tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:15:22.331441] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:22.332550] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:22.340116] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.343181] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.345129] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.345216] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.353516] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.354669] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.355812] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:22.355834] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:22.355851] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.355863] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1003}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:22.355971] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=84] build ls locality stat map(tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:15:22.355979] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.355992] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.355999] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:22.356010] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:22.356090] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.356105] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.356114] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.356120] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:22.356128] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3737, tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:15:22.356138] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:22.357262] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=9] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=95349, tenant_id=1014, acc_dr_task=0) [2025-08-15 10:15:22.390967] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A16-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.391224] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8837-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.393490] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A16-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:22.393509] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A16-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.393739] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8837-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:22.393757] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8837-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.412832] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:22.414059] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224122412755, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:15:22.414082] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224127414079, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:15:22.438490] INFO [RS] finish (ob_mview_maintenance_task.cpp:280) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=12] mview maintenace task finish(tenant_id_=1008, round_=105550, status_=100, error_code_=0, cost_us_=20001399, prepare_cost_us_=0, gc_mview_cost_us_=1238, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:15:22.439670] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C685B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.459416] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:22.459428] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:22.459433] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:22.499392] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0A2-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.499411] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E711-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.504181] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF20C-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.520228] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1275-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81835938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82226562GB}) [2025-08-15 10:15:22.520268] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1275-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:15:22.531328] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224120978446005, replayable_scn:1755224120978446005, readable_scn:1755224120978446005, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224122527680, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224120978446005, replayable_scn:1755224120978446005, readable_scn:1755224120978446005, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224122527680, ora_rowscn_=1755224122341503000, content_changed=true) [2025-08-15 10:15:22.531520] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:22.531532] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:22.531540] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:22.532031] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:22.532041] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:22.532047] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:22.533349] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:22.537494] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:22.537506] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:22.537512] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:22.543484] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:22.543497] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:22.543502] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:22.544784] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:22.548762] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:22.548773] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:22.548778] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:22.586464] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AE-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:22.588966] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AE-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:22.668332] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9590-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.670887] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9590-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:22.670903] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9590-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.686271] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885C-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.688352] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C685E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.688862] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885C-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:22.688878] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885C-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.698547] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835E-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.699644] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD70-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.701302] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:340) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835E-0-0] [lt=14] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1014, ls_id:{id:1}, sync_scn:{val:1755224122302583004, v:0}, readable_scn:{val:1755224122302583004, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-08-15 10:15:22.701632] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835E-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.703555] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.703667] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:22.703705] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:459) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835E-0-0] [lt=7] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1755224120299212004, v:0}) [2025-08-15 10:15:22.705628] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:22.705641] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:22.705649] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:22.709483] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D0-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:22.710215] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5245-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:22.712029] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D0-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:22.713823] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5245-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:22.715320] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB23-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.717844] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB23-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:22.717862] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB23-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:22.748908] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD7F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.748913] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF049-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.782461] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB867B-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:22.785023] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB867B-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:22.785642] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E4-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:22.788102] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E4-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:22.791212] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:22.792453] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224122791138, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:15:22.792481] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224127792479, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:15:22.792869] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:22.794097] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224122792800, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:15:22.794118] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224127794116, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:15:22.846595] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8CFF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.872081] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8A-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:22.873267] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8A-0-0] [lt=12] log restore source is empty, just skip(ret=-4018, user_tenant_id=1014) [2025-08-15 10:15:22.875776] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8A-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:22.881050] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B359-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:22.906190] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=24] start do ObArchiveSchedulerService round(round=101219) [2025-08-15 10:15:22.907504] INFO [ARCHIVE] checkpoint_ (ob_tenant_archive_scheduler.cpp:682) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=10] schedule archive(round_info={key:{tenant_id:1016, dest_no:0}, incarnation:1, dest_id:1001, round_id:1, state:{status:"DOING"}, start_scn:{val:1752091205985713000, v:0}, checkpoint_scn:{val:1755224005638556002, v:0}, max_scn:{val:1755224059891515001, v:0}, compatible:{version:1}, base_piece_id:1, used_piece_id:37, piece_switch_interval:86400000000, frozen_input_bytes:44838045649, frozen_output_bytes:44838045649, active_input_bytes:331266514, active_output_bytes:331266514, deleted_input_bytes:0, deleted_output_bytes:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com", comment:""}) [2025-08-15 10:15:22.908949] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727237][T1015_ArchiveSv][T1016][YB420ACB0122-000633A34F144D2E-0-0] [lt=19] get number(ret=0, service_type_=0, range=1, base_id=1755224122908158000, start_id=1755224122908158000, end_id=1755224122908158001) [2025-08-15 10:15:22.908967] INFO [SHARE] get_backup_scn (ob_backup_struct.cpp:2155) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=14] get tenant gts(ret=0, ret="OB_SUCCESS", tenant_id=1016, scn={val:1755224122908158000, v:0}) [2025-08-15 10:15:22.908984] INFO [SHARE] count_ (ob_archive_checkpoint.cpp:173) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=9] print count result(ret=0, counter={ls_count:4, deleted_ls_count:0, not_start_cnt:0, interrupted_cnt:0, doing_cnt:4, stopped_cnt:0, interrupted_ls_id:{id:-1}, max_scn:{val:1755224059891515001, v:0}, checkpoint_scn:{val:1755224005638556002, v:0}, max_active_piece_id:37}) [2025-08-15 10:15:22.910159] INFO [SHARE] get_backup_dest (ob_backup_connectivity.cpp:1268) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=11] success get backup dest(sql=SELECT authorization,extension FROM __all_backup_storage_info WHERE tenant_id = 1016 AND path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog' AND endpoint = 'host=cos.ap-beijing.myqcloud.com', tenant_id=1016, backup_dest={root_path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog", storage_info:{endpoint:"host=cos.ap-beijing.myqcloud.com", access_id:"access_id=AKIDiKl3A5ZXJ959EMuL3hJ1g9PTM5lFvjhT", extension:"appid=1254236265&checksum_type=md5", type:"COS", checksum_type:1, max_iops:0, max_bandwidth:0, role_arn:0x7f1bf396af90, external_id:0x7f1bf396b090}}) [2025-08-15 10:15:22.911498] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB628E-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:22.913985] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB628E-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:22.947775] INFO [SHARE] get_backup_dest (ob_backup_connectivity.cpp:1268) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=17] success get backup dest(sql=SELECT authorization,extension FROM __all_backup_storage_info WHERE tenant_id = 1016 AND path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog' AND endpoint = 'host=cos.ap-beijing.myqcloud.com', tenant_id=1016, backup_dest={root_path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog", storage_info:{endpoint:"host=cos.ap-beijing.myqcloud.com", access_id:"access_id=AKIDiKl3A5ZXJ959EMuL3hJ1g9PTM5lFvjhT", extension:"appid=1254236265&checksum_type=md5", type:"COS", checksum_type:1, max_iops:0, max_bandwidth:0, role_arn:0x7f1e413baf90, external_id:0x7f1e413bb090}}) [2025-08-15 10:15:22.949139] INFO [SHARE] do_compare_and_swap_ (ob_inner_table_operator.cpp:937) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=20] compare and swap one column(key={key:{tenant_id:1016, dest_no:0}, incarnation:1, dest_id:1001, round_id:1, state:{status:"DOING"}, start_scn:{val:1752091205985713000, v:0}, checkpoint_scn:{val:1755224005638556002, v:0}, max_scn:{val:1755224059891515001, v:0}, compatible:{version:1}, base_piece_id:1, used_piece_id:37, piece_switch_interval:86400000000, frozen_input_bytes:44838045649, frozen_output_bytes:44838045649, active_input_bytes:331266514, active_output_bytes:331266514, deleted_input_bytes:0, deleted_output_bytes:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com", comment:""}, assignments="tenant_id = 1016, dest_no = 0, round_id = 1, incarnation = 1, dest_id = 1001, status = 'DOING', start_scn = 1752091205985713000, checkpoint_scn = 1755224005638556002, max_scn = 1755224059891515001, compatible = 1, base_piece_id = 1, used_piece_id = 37, piece_switch_interval = 86400000000, frozen_input_bytes = 44838045649, frozen_output_bytes = 44838045649, active_input_bytes = 331266514, active_output_bytes = 331266514, deleted_input_bytes = 0, deleted_output_bytes = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com', comment = ''", predicates="status = 'DOING' and round_id = 1 and checkpoint_scn = 1755224005638556002", affected_rows=0, sql=update __all_log_archive_progress set tenant_id = 1016, dest_no = 0, round_id = 1, incarnation = 1, dest_id = 1001, status = 'DOING', start_scn = 1752091205985713000, checkpoint_scn = 1755224005638556002, max_scn = 1755224059891515001, compatible = 1, base_piece_id = 1, used_piece_id = 37, piece_switch_interval = 86400000000, frozen_input_bytes = 44838045649, frozen_output_bytes = 44838045649, active_input_bytes = 331266514, active_output_bytes = 331266514, deleted_input_bytes = 0, deleted_output_bytes = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com', comment = '' where tenant_id = 1016 AND dest_no = 0 and status = 'DOING' and round_id = 1 and checkpoint_scn = 1755224005638556002) [2025-08-15 10:15:22.949298] WDIAG [STORAGE.TRANS] mvcc_sanity_check_ (ob_mvcc_row.cpp:937) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=28][errcode=-5024] find double insert node(ret=-5024, node=this=0x7f1d4d477b58 trans_version={val:4611686018427387903, v:0} scn={val:4611686018427387903, v:0} tx_id={txid:86701926} prev=(nil) next=(nil) modify_count=4294967295 acc_checksum=0 version=1755222868003467 type=0 flag=128 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:6} write_epoch=2, prev=this=0x7f1d4d477900 trans_version={val:1755224072901249001, v:0} scn={val:1755224072901249001, v:0} tx_id={txid:86701229} prev=0x7f1d4d477780 next=(nil) modify_count=43 acc_checksum=416833582 version=1755222868003467 type=0 flag=4 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:1055} write_epoch=4, snapshot_version={val:1755224122949211000, v:0}, *this={this=0x7f1d4d466918 latch_=locked flag=11 first_dml=DELETE last_dml=INSERT update_since_compact=2 list_head=0x7f1d4d477900 latest_compact_node=0x7f1d4d4773a0 max_trans_version={val:1755224072901249001, v:0} max_trans_id=86701229 max_elr_trans_version={val:1755224072901249001, v:0} max_elr_trans_id=86701229 latest_compact_ts=1755224022720472 last_compact_cnt=41 total_trans_node_cnt=44 max_modify_scn={val:1755224072901249001, v:0} min_modify_scn={val:1755222868004408000, v:0}}) [2025-08-15 10:15:22.949335] WDIAG [STORAGE.TRANS] mvcc_write (ob_mvcc_row.cpp:999) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=29][errcode=-5024] mvcc write failed(ret=-5024, node=this=0x7f1d4d477b58 trans_version={val:4611686018427387903, v:0} scn={val:4611686018427387903, v:0} tx_id={txid:86701926} prev=(nil) next=(nil) modify_count=4294967295 acc_checksum=0 version=1755222868003467 type=0 flag=128 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:6} write_epoch=2, ctx={this:0x7f1deadc2dd0, ls_id:{id:1}, ls:0x7f1c55fba150, branch:0, timeout:1755224152849194, tablet_id:{id:350}, table_iter:0x7f1deadc1308, table_version:1750151411699448, mvcc_acc_ctx:{type:3, abs_lock_timeout_ts:1755224152849194, tx_lock_timeout_us:-1, snapshot:{version:{val:1755224122949211000, v:0}, tx_id:{txid:86701926}, scn:{branch:0, seq:4}}, tx_table_guards:{tx_table_guard:{tx_table:0x7f1c55fc5b10, epoch:1, mini_cache_:[0:{}, 1:{}, 2:{}, 3:{}]}, src_tx_table_guard:{tx_table:null, epoch:-1, mini_cache_:[0:{}, 1:{}, 2:{}, 3:{}]}, src_ls_handle:{ls_map_:null, ls_:null, mod_:-1}}, tx_id:{txid:86701926}, tx_desc:{this:0x7f1e7ab4c660, tx_id:{txid:86701926}, state:2, addr:"10.203.1.34:2882", tenant_id:1015, session_id:3221647594, assoc_session_id:3221647594, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:{branch:0, seq:1}, op_sn:4, alloc_ts:1755224122948157, active_ts:1755224122948157, commit_ts:-1, finish_ts:-1, timeout_us:29999994, lock_timeout_us:-1, expire_ts:1755224152948151, coord_id:{id:-1}, parts:[{id:{id:1}, addr:"10.203.1.34:2882", epoch:509118134799206, first_scn:{branch:0, seq:3}, last_scn:{branch:0, seq:3}, last_touch_ts:1755224122947415}], exec_info_reap_ts:1755224122947414, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:1727264618, cluster_version:17180067073, seq_base:1755224122947411, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, conflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, modified_tables:[], last_rc_snapshot_version:{val:1755224122949211000, v:0}, ref:2}, tx_ctx:0x7f1c9defc7d0, mem_ctx:0x7f1c9defdd50, tx_scn:{branch:0, seq:6}, write_flag:{is_table_api:0, is_table_lock:0, is_mds:0, is_dml_batch_opt:0, is_insert_up:1, is_write_only_index:0, is_check_row_locked:0, is_lob_aux:0, is_skip_flush_redo:0, is_update_uk:0, is_update_pk_dop:0}, handle_start_time:-1, is_standby_read:false, lock_wait_start_ts:0, is_inited:true}, tablet_stat:{ls_id:0, tablet_id:0, query_cnt:0, merge_cnt:0, scan_logical_row_cnt:0, scan_physical_row_cnt:0, scan_micro_block_cnt:0, pushdown_micro_block_cnt:0, exist_row_total_table_cnt:0, exist_row_read_table_cnt:0, insert_row_cnt:0, update_row_cnt:0, delete_row_cnt:0}, is_read_store_ctx:false, update_full_column:false}) [2025-08-15 10:15:22.949405] WDIAG [STORAGE.TRANS] mvcc_write (ob_mvcc_engine.cpp:333) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=56][errcode=-5024] mvcc write failed(ret=-5024, arg={tx_id:{txid:86701926}, data:0x7f1deadbfcc0, old_row:null, modify_count:4294967295, acc_checksum:0, memstore_version:1755222868003467, seq_no:{branch:0, seq:6}, write_epoch:2, scn:{val:4611686018427387903, v:0}, column_cnt:23}) [2025-08-15 10:15:22.949419] WDIAG [STORAGE.TRANS] on_key_duplication_retry (ob_memtable_context.cpp:314) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=11][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:22.949442] WDIAG [STORAGE] insert_row_without_rowkey_check (ob_tablet.cpp:4643) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=18][errcode=-5024] fail to set memtable(ret=-5024) [2025-08-15 10:15:22.949453] WDIAG [STORAGE] insert_row_to_tablet (ob_ls_tablet_service.cpp:5667) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=8][errcode=-5024] failed to write table row(ret=-5024, table id=350, col_descs=[cnt:23, column_id=18 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=19 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=20 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=21 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=16 {type:"TIMESTAMP", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=17 {type:"TIMESTAMP", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=22 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=23 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=24 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=25 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=26 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=27 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=28 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=29 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=30 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=31 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=32 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=33 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=34 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=35 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=36 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=37 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=38 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0], datum_row={row_flag:{flag:"INSERT", flag_type:0}, trans_id:{txid:0}, scan_index:0, mvcc_row_flag:{first:0, uncommitted:0, shadow:0, compact:0, ghost:0, last:0, reserved:0, flag:0}, snapshot_version:0, fast_filter_skipped:false, have_uncommited_row:false, group_idx:0, count:23, datum_buffer:{capacity:23, datums:0x7f18777e5ea0, local_datums:0x7f18777e5af0}, datums:[col_id=0:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e58a4, hex: F803000000000000, int: 1016}, col_id=1:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e58ac, hex: E903000000000000, int: 1001}, col_id=2:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e58b4, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=3:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e58bc, hex: 2500000000000000, int: 37}, col_id=4:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e58c4, hex: 5B7E59F65D3C0600, int: 1755224122949211}, col_id=5:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e58cc, hex: 5B7E59F65D3C0600, int: 1755224122949211}, col_id=6:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e58d4, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=7:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e58dc, hex: 0000000000000000, int: 0}, col_id=8:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e58e4, hex: 0000000000000000, int: 0}, col_id=9:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e58ec, hex: 683BF9A98FBA5B18, int: 1755201605985713000}, col_id=10:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e58f4, hex: 620953FDEECE5B18, int: 1755224005638556002}, col_id=11:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e58fc, hex: 797E0D9FFBCE5B18, int: 1755224059891515001}, col_id=12:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e5904, hex: 683B483B24095C18, int: 1755288005985713000}, col_id=13:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e590c, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=14:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e5914, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=15:{len: 4, flag: 0, null: 0, ptr: 0x7f18777e591c, hex: 6E6F6E65, cstr: none}, col_id=16:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e5920, hex: D2B9BE1300000000, int: 331266514}, col_id=17:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e5928, hex: D2B9BE1300000000, int: 331266514}, col_id=18:{len: 6, flag: 0, null: 0, ptr: 0x7f18777e5930, hex: 414354495645, cstr: ACTIVE}, col_id=19:{len: 9, flag: 0, null: 0, ptr: 0x7f18777e5936, hex: 415641494C41424C45, cstr: AVAILABLE}, col_id=20:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e593f, hex: 0000000000000000, int: 0}, col_id=21:{len: 8, flag: 0, null: 0, ptr: 0x7f18777e5947, hex: 0000000000000000, int: 0}, col_id=22:{len: 135, flag: 0, null: 0, ptr: 0x7f18777e594f, hex: 636F733A2F2F6261636B75702D6F6365616E626173652D313235343233363236352F6261636B75702D6F62636C75737465722F6F625F766563746F722F313732373236343631382F74656E616E745F696E6361726E6174696F6E5F312F313031362F636C6F673F686F73743D636F732E61702D6265696A696E672E6D7971636C, cstr: cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcl}]}) [2025-08-15 10:15:22.949536] WDIAG [STORAGE] insert_row (ob_ls_tablet_service.cpp:3241) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=54][errcode=-5024] failed to write row(ret=-5024) [2025-08-15 10:15:22.949564] INFO [STORAGE.TRANS] rollback (ob_memtable_context.cpp:851) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=7] memtable handle rollback to successfuly(from_seq_no={branch:0, seq:7}, to_seq_no={branch:0, seq:5}, remove_cnt=1, elapsed=0, this={ObIMvccCtx={alloc_type=0 ctx_descriptor=0 min_table_version=1750151411699400 max_table_version=1750151411699448 trans_version={val:4611686018427387903, v:0} commit_version={val:0, v:0} lock_wait_start_ts=0 replay_compact_version={val:0, v:0}} end_code=0 tx_status=0 is_readonly=false ref=0 trans_id={txid:86701926} ls_id=1 row_callback[alloc:1, free:0, unsubmit:1] redo[fill:0,sync_succ:0, sync_fail:0] main_list_len=2 pending_log_size=41 callback_list:{cnt=1 stat:[tx_end=0, rollback_to=1, fast_commit=0, remove_memtable=0, ext_info_log=0] detail:[(log_epoch,length,logged,synced,appended,removed,unlog_removed,branch_removed)|0:(1,2,0,0,3,1,1,0)|]}}) [2025-08-15 10:15:22.949691] INFO [SHARE] do_insert_or_update_row_ (ob_inner_table_operator.cpp:699) [727237][T1015_ArchiveSv][T1015][YB420ACB0122-000633A34F144D2E-0-0] [lt=14] insert/update one row(row={key:{tenant_id:1016, dest_id:1001, round_id:1, piece_id:37}, incarnation:1, dest_no:0, file_count:0, start_scn:{val:1755201605985713000, v:0}, checkpoint_scn:{val:1755224005638556002, v:0}, max_scn:{val:1755224059891515001, v:0}, end_scn:{val:1755288005985713000, v:0}, compatible:{version:1}, input_bytes:331266514, output_bytes:331266514, status:{status:"ACTIVE"}, file_status:0, cp_file_id:0, cp_file_offset:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com"}, affected_rows=0, sql=INSERT INTO __all_log_archive_piece_files (tenant_id, dest_id, round_id, piece_id, incarnation, dest_no, file_count, start_scn, checkpoint_scn, max_scn, end_scn, compatible, input_bytes, output_bytes, status, file_status, cp_file_id, cp_file_offset, path) VALUES (1016, 1001, 1, 37, 1, 0, 0, 1755201605985713000, 1755224005638556002, 1755224059891515001, 1755288005985713000, 1, 331266514, 331266514, 'ACTIVE', 'AVAILABLE', 0, 0, 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com') ON DUPLICATE KEY UPDATE incarnation = 1, dest_no = 0, file_count = 0, start_scn = 1755201605985713000, checkpoint_scn = 1755224005638556002, max_scn = 1755224059891515001, end_scn = 1755288005985713000, compatible = 1, input_bytes = 331266514, output_bytes = 331266514, status = 'ACTIVE', file_status = 'AVAILABLE', cp_file_id = 0, cp_file_offset = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1016/clog?host=cos.ap-beijing.myqcloud.com') [2025-08-15 10:15:22.954623] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:22.955876] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224122954547, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:15:22.955902] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224127955899, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:15:22.989921] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58CA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:22.990625] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9AD-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.040417] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:23.041664] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224123040339, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:15:23.041688] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224128041686, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:15:23.102382] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58C4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.122190] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [438126][T1005_ArchiveSv][T1005][YB420ACB0122-000633820735554B-0-0] [lt=20] start do ObArchiveSchedulerService round(round=168832) [2025-08-15 10:15:23.123489] INFO [ARCHIVE] checkpoint_ (ob_tenant_archive_scheduler.cpp:682) [438126][T1005_ArchiveSv][T1005][YB420ACB0122-000633820735554B-0-0] [lt=13] schedule archive(round_info={key:{tenant_id:1006, dest_no:0}, incarnation:1, dest_id:1001, round_id:1, state:{status:"STOP"}, start_scn:{val:1748944774947721000, v:0}, checkpoint_scn:{val:1752037015224032000, v:0}, max_scn:{val:1752037085107831001, v:0}, compatible:{version:1}, base_piece_id:1, used_piece_id:36, piece_switch_interval:86400000000, frozen_input_bytes:16367268487, frozen_output_bytes:16367268487, active_input_bytes:0, active_output_bytes:0, deleted_input_bytes:0, deleted_output_bytes:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1006/clog?host=cos.ap-beijing.myqcloud.com", comment:""}) [2025-08-15 10:15:23.134678] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0A5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.147136] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF046-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.183834] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CFD-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.198467] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9B2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.203670] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF55-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.203672] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FD-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.206196] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF55-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:23.206196] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FD-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:23.206212] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF55-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.206211] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FD-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.283051] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D0-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224122804206003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224122804206004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224122804206004, v:0}}], last_dump_ts_=1755224118251805, force_dump=false) [2025-08-15 10:15:23.283072] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D0-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.283158] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AF-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.283344] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C05-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.283374] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC037-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.283793] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.286261] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D0-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224122804206004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224122804206004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224122804206004, v:0}}], last_dump_ts_=1755224118254947, force_dump=false) [2025-08-15 10:15:23.286298] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D0-0-0] [lt=26] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.286436] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C05-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.286442] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC037-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.286455] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AF-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.286723] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.288411] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D0-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.288462] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C05-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.288463] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC037-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.288512] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D0-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.288558] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C05-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.288573] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AF-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.288602] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC037-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.288700] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413AF-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.288720] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.288802] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.346190] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.349466] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.351342] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.351448] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.396254] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A17-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.396473] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8838-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.398772] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A17-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:23.398790] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A17-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.399021] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8838-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:23.399038] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8838-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.439743] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C685C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.459242] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224121301358003, replayable_scn:1755224121301358003, readable_scn:1755224121301358003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224123455942, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224121301358003, replayable_scn:1755224121301358003, readable_scn:1755224121301358003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224123455942, ora_rowscn_=1755224122700218000, content_changed=true) [2025-08-15 10:15:23.459413] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:23.459425] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:23.459431] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:23.459442] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1014, cost_time_us=177, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:23.460349] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A08CA-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.93164062GB}) [2025-08-15 10:15:23.460408] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A08CA-0-0] [lt=36] [DDL] execute ddl like stmt(ret=0, cost=64, ddl_arg=NULL) [2025-08-15 10:15:23.499467] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0A3-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.499487] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E712-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.531504] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:23.531515] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:23.531520] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:23.531884] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224121801772005, replayable_scn:1755224121801772005, readable_scn:1755224121801772005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224123528547, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224121801772005, replayable_scn:1755224121801772005, readable_scn:1755224121801772005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224123528547, ora_rowscn_=1755224123284694000, content_changed=true) [2025-08-15 10:15:23.532057] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:23.532077] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:23.532084] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:23.537534] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:23.537553] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:23.537559] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:23.538949] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:23.543429] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224121801772005, replayable_scn:1755224121801772005, readable_scn:1755224121801772005, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224123540054, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224121801772005, replayable_scn:1755224121801772005, readable_scn:1755224121801772005, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224123540054, ora_rowscn_=1755224123284694000, content_changed=true) [2025-08-15 10:15:23.543588] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:23.543598] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:23.543606] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:23.548694] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224121801772006, replayable_scn:1755224121801772006, readable_scn:1755224121801772006, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224123545282, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224121801772006, replayable_scn:1755224121801772006, readable_scn:1755224121801772006, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224123545282, ora_rowscn_=1755224123284694000, content_changed=true) [2025-08-15 10:15:23.548855] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:23.548866] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:23.548872] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:23.594304] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AF-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:23.596848] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38AF-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:23.598354] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:23.599599] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224123598316, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:15:23.599622] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224128599620, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:15:23.673645] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9591-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.673918] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC633-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.676164] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9591-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:23.676181] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9591-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.688430] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C685F-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.691563] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885D-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.694115] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885D-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:23.694132] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.699716] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD71-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.704636] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835F-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.705521] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224121801772005, replayable_scn:1755224121801772005, readable_scn:1755224121801772005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224123702118, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224121801772005, replayable_scn:1755224121801772005, readable_scn:1755224121801772005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224123702118, ora_rowscn_=1755224123285750000, content_changed=true) [2025-08-15 10:15:23.705702] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:23.705721] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:23.705731] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:23.707648] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835F-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224123304727003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224123304727003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224123304727003, v:0}}], last_dump_ts_=1755224118676894, force_dump=false) [2025-08-15 10:15:23.707671] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835F-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.709530] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835F-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.709627] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D835F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:23.717388] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D1-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:23.719081] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5246-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:23.719897] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D1-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:23.720588] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB24-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.721544] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5246-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:23.723130] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB24-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:23.723148] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB24-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:23.748945] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF04A-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.748946] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD80-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.790302] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB867C-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:23.792749] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB867C-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:23.793341] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E5-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:23.794518] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E5-0-0] [lt=12] log restore source is empty, just skip(ret=-4018, user_tenant_id=1016) [2025-08-15 10:15:23.796992] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E5-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:23.810588] INFO [RS.LB] idle (ob_root_balancer.cpp:87) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=30] root balance idle(idle_time=10000000) [2025-08-15 10:15:23.810602] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:70) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=9] start check unit migrate finish [2025-08-15 10:15:23.810615] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=6] try check migrate unit finish by tenant(tenant_id=1) [2025-08-15 10:15:23.819768] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=6] try check migrate unit finish by tenant(tenant_id=1004) [2025-08-15 10:15:23.845102] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=12] try check migrate unit finish by tenant(tenant_id=1006) [2025-08-15 10:15:23.846666] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D00-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.870489] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1008) [2025-08-15 10:15:23.881193] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8B-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:23.882427] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B35A-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:23.883732] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8B-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:23.896169] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1012) [2025-08-15 10:15:23.919325] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB628F-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:23.921618] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1014) [2025-08-15 10:15:23.923149] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB628F-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:23.946976] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1016) [2025-08-15 10:15:23.972535] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1018) [2025-08-15 10:15:23.990007] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58CB-0-0] [lt=22] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.990702] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9AE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:23.998124] INFO [RS] check (ob_alter_locality_finish_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BB-0-0] [lt=12] start to check alter locality finish [2025-08-15 10:15:23.999265] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BC-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.000427] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BC-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.002741] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BE-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.003875] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814BE-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.006152] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814C0-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.007283] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814C0-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.009553] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814C2-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.010690] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814C2-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.013000] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814C4-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.014124] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814C4-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.016382] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814C6-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.017506] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814C6-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.019768] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814C8-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.020899] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814C8-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.023181] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CA-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.024303] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CA-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.025484] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CA-0-0] [lt=11] start check shrink resource pool [2025-08-15 10:15:24.025500] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CB-0-0] [lt=9] start check shrink resource pool(tenant_id=1) [2025-08-15 10:15:24.026601] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CB-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.027742] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CB-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.027789] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CC-0-0] [lt=10] start check shrink resource pool(tenant_id=1004) [2025-08-15 10:15:24.028883] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CC-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.030017] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CC-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.030064] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CD-0-0] [lt=11] start check shrink resource pool(tenant_id=1006) [2025-08-15 10:15:24.031181] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CD-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.032314] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CD-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.032361] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CE-0-0] [lt=11] start check shrink resource pool(tenant_id=1008) [2025-08-15 10:15:24.033479] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CE-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.034610] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CE-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.034657] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CF-0-0] [lt=10] start check shrink resource pool(tenant_id=1012) [2025-08-15 10:15:24.035756] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CF-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.036900] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814CF-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.036955] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D0-0-0] [lt=10] start check shrink resource pool(tenant_id=1014) [2025-08-15 10:15:24.038048] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D0-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.039169] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D0-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.039216] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D1-0-0] [lt=11] start check shrink resource pool(tenant_id=1016) [2025-08-15 10:15:24.040320] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D1-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.041456] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D1-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.041503] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D2-0-0] [lt=10] start check shrink resource pool(tenant_id=1018) [2025-08-15 10:15:24.042605] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D2-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.043732] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D2-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.043778] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:99) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D2-0-0] [lt=10] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.043789] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D2-0-0] [lt=8] start check primary zone [2025-08-15 10:15:24.043799] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D3-0-0] [lt=6] start check primary zone(tenant_id=1) [2025-08-15 10:15:24.044886] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D3-0-0] [lt=5][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.044931] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D4-0-0] [lt=10] start check primary zone(tenant_id=1004) [2025-08-15 10:15:24.046032] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D4-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.046080] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D5-0-0] [lt=11] start check primary zone(tenant_id=1006) [2025-08-15 10:15:24.047175] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D5-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.047221] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D6-0-0] [lt=10] start check primary zone(tenant_id=1008) [2025-08-15 10:15:24.048348] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D6-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.048396] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D7-0-0] [lt=11] start check primary zone(tenant_id=1012) [2025-08-15 10:15:24.049498] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D7-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.049552] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D8-0-0] [lt=11] start check primary zone(tenant_id=1014) [2025-08-15 10:15:24.050653] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D8-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.050700] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D9-0-0] [lt=11] start check primary zone(tenant_id=1016) [2025-08-15 10:15:24.051808] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814D9-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.051856] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=10] start check primary zone(tenant_id=1018) [2025-08-15 10:15:24.052965] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:24.053012] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:166) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=10] start do unit balance [2025-08-15 10:15:24.059119] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:350) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=7] start distribute for server status change [2025-08-15 10:15:24.059219] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:401) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=12] finish distribute for server status change(ret=0) [2025-08-15 10:15:24.059230] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:200) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=9] finish do unit balance(ret=0) [2025-08-15 10:15:24.065036] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:115) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=4] start to do tenant group unit balance [2025-08-15 10:15:24.065078] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=12] start to do tenant group unit balance(zone=zone1) [2025-08-15 10:15:24.065091] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=8] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.203.1.34:2882", disk_in_use_:3147825152, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:24.065110] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=11] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:24.065138] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=12] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:24.065155] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=8] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:24.065172] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=10] start do non-tenantgroup unit balance task(zone=zone1) [2025-08-15 10:15:24.065183] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=7] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:24.065206] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=14] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.93164062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.1.34:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.93164062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:24.065249] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=34] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.203.1.34:2882", disk_in_use_:3147825152, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:24.065261] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=10] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.1.34:2882", disk_in_use_:3147825152, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:24.065277] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=10] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:24.065293] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=12] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1006, unit_config:{unit_config_id:1005, name:"config_opsone_zone1_s_2_4_ywx", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7988, unit_config:0x7f1e9aa71c38, pool:0x7f1c3beb2f18}, {unit_id:1008, tenant_id:1006, resource_pool_id:1008, unit_config:{unit_config_id:1008, name:"config_test_zone1_s_2_4_emp", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f77d8, unit_config:0x7f1e9aa71998, pool:0x7f1c3beb27a8}, {unit_id:1011, tenant_id:1008, resource_pool_id:1011, unit_config:{unit_config_id:1011, name:"config_mcp_zone1_s_4_8_tmc", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7550, unit_config:0x7f1e9aa716f8, pool:0x7f1e443fda40}, {unit_id:1016, tenant_id:1012, resource_pool_id:1016, unit_config:{unit_config_id:1017, name:"config_oms_zone1_s_4_8_gzp", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f73a0, unit_config:0x7f1e9aa71458, pool:0x7f1e443fd2d0}, {unit_id:1019, tenant_id:1014, resource_pool_id:1019, unit_config:{unit_config_id:1019, name:"config_grafana_test_zone1_s_1_2_pri", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7118, unit_config:0x7f1e9aa71298, pool:0x7f1e443fc7a8}, {unit_id:1024, tenant_id:1016, resource_pool_id:1024, unit_config:{unit_config_id:1023, name:"config_temporalio_zone1_s_2_4_ykw", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6ce0, unit_config:0x7f1e9aa70f18, pool:0x7f1e74dff2d0}, {unit_id:1026, tenant_id:1018, resource_pool_id:1026, unit_config:{unit_config_id:1027, name:"config_media_zone1_s_2_4_kmn", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6b30, unit_config:0x7f1e9aa70b98, pool:0x7f1e74dfeb60}], not_grant_units=[], available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:24.065352] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=35] start to do tenant group unit balance(zone=zone2) [2025-08-15 10:15:24.065361] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.203.4.38:2882", disk_in_use_:3131047936, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:24.065374] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:24.065391] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=9] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:24.065403] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=7] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:24.065412] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone2) [2025-08-15 10:15:24.065417] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:24.065430] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.38:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:24.065462] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=25] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.203.4.38:2882", disk_in_use_:3131047936, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:24.065468] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=5] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.38:2882", disk_in_use_:3131047936, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:24.065479] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:24.065490] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=9] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1004, unit_config:{unit_config_id:1004, name:"config_opsone_zone2_s_2_4_wag", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7b38, unit_config:0x7f1e9aa71d18, pool:0x7f1c3beb3688}, {unit_id:1009, tenant_id:1006, resource_pool_id:1009, unit_config:{unit_config_id:1009, name:"config_test_zone2_s_2_4_yey", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7700, unit_config:0x7f1e9aa718b8, pool:0x7f1c3beb23f0}, {unit_id:1010, tenant_id:1008, resource_pool_id:1010, unit_config:{unit_config_id:1010, name:"config_mcp_zone2_s_4_8_bjb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7628, unit_config:0x7f1e9aa717d8, pool:0x7f1c3beb2038}, {unit_id:1018, tenant_id:1012, resource_pool_id:1018, unit_config:{unit_config_id:1016, name:"config_oms_zone2_s_4_8_kbx", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f71f0, unit_config:0x7f1e9aa71538, pool:0x7f1e443fcb60}, {unit_id:1021, tenant_id:1014, resource_pool_id:1021, unit_config:{unit_config_id:1021, name:"config_grafana_test_zone2_s_1_2_yyc", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f6f68, unit_config:0x7f1e9aa710d8, pool:0x7f1e443fc038}, {unit_id:1023, tenant_id:1016, resource_pool_id:1023, unit_config:{unit_config_id:1022, name:"config_temporalio_zone2_s_2_4_lsv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6db8, unit_config:0x7f1e9aa70ff8, pool:0x7f1e74dff688}, {unit_id:1027, tenant_id:1018, resource_pool_id:1027, unit_config:{unit_config_id:1025, name:"config_media_zone2_s_2_4_yzd", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6a58, unit_config:0x7f1e9aa70d58, pool:0x7f1e74dfe7a8}], not_grant_units=[], available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:24.065541] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=31] start to do tenant group unit balance(zone=zone3) [2025-08-15 10:15:24.065550] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.203.4.39:2882", disk_in_use_:3030384640, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:24.065562] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:24.065578] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=9] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:24.065590] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=7] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:24.065598] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone3) [2025-08-15 10:15:24.065603] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:24.065616] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=10] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82226562GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.39:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82226562GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:24.065647] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=24] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.203.4.39:2882", disk_in_use_:3030384640, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:24.065652] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=5] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.39:2882", disk_in_use_:3030384640, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:24.065662] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=7] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:24.065677] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=11] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1005, unit_config:{unit_config_id:1006, name:"config_opsone_zone3_s_2_4_vxv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7a60, unit_config:0x7f1e9aa71b58, pool:0x7f1c3beb32d0}, {unit_id:1007, tenant_id:1006, resource_pool_id:1007, unit_config:{unit_config_id:1007, name:"config_test_zone3_s_2_4_mui", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f78b0, unit_config:0x7f1e9aa71a78, pool:0x7f1c3beb2b60}, {unit_id:1012, tenant_id:1008, resource_pool_id:1012, unit_config:{unit_config_id:1012, name:"config_mcp_zone3_s_4_8_seu", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7478, unit_config:0x7f1e9aa71618, pool:0x7f1e443fd688}, {unit_id:1017, tenant_id:1012, resource_pool_id:1017, unit_config:{unit_config_id:1018, name:"config_oms_zone3_s_4_8_nvb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f72c8, unit_config:0x7f1e9aa71378, pool:0x7f1e443fcf18}, {unit_id:1020, tenant_id:1014, resource_pool_id:1020, unit_config:{unit_config_id:1020, name:"config_grafana_test_zone3_s_1_2_xbz", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7040, unit_config:0x7f1e9aa711b8, pool:0x7f1e443fc3f0}, {unit_id:1022, tenant_id:1016, resource_pool_id:1022, unit_config:{unit_config_id:1024, name:"config_temporalio_zone3_s_2_4_rxz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6e90, unit_config:0x7f1e9aa70e38, pool:0x7f1e74dffa40}, {unit_id:1025, tenant_id:1018, resource_pool_id:1025, unit_config:{unit_config_id:1026, name:"config_media_zone3_s_2_4_ukz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6c08, unit_config:0x7f1e9aa70c78, pool:0x7f1e74dfef18}], not_grant_units=[], available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:24.102456] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58C5-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.134782] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0A6-0-0] [lt=38] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.147211] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF047-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.183908] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CFE-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.198541] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9B3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.208902] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF56-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.208915] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FE-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.211457] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FE-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:24.211474] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FE-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.212171] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF56-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:24.212192] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF56-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.240164] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:24.240181] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:24.246140] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:24.247327] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:24.271375] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D2E-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}) [2025-08-15 10:15:24.271417] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D2E-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:15:24.289526] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D1-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.289607] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC038-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.289616] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C06-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224123805200002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224124240206000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224124240206000, v:0}}], last_dump_ts_=1755224119257895, force_dump=false) [2025-08-15 10:15:24.289639] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C06-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.289675] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B0-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.289835] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783F-0-0] [lt=11] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224123805200004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224123805200004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224123805200004, v:0}}], last_dump_ts_=1755224119259420, force_dump=false) [2025-08-15 10:15:24.289862] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783F-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.292746] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B0-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.292765] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C06-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224123805200004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224123805200005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224123805200005, v:0}}], last_dump_ts_=1755224119260946, force_dump=false) [2025-08-15 10:15:24.292792] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C06-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.292838] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC038-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.292902] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.293056] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D1-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.294838] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C06-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.294944] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C06-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.294957] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC038-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.295081] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC038-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.295110] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D1-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.295208] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D1-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.295332] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B0-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.295436] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B0-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.295512] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783F-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.295595] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB783F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.352514] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762F-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224124222335000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224124222335000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224124222335000, v:0}}], last_dump_ts_=1755224119321220, force_dump=false) [2025-08-15 10:15:24.352538] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762F-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.355508] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762F-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224124222335000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224124352755000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224124352755000, v:0}}], last_dump_ts_=1755224119324327, force_dump=false) [2025-08-15 10:15:24.355531] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762F-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.357456] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762F-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.357540] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D762F-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.401556] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A18-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.401846] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8839-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.404099] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A18-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:24.404114] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A18-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.404521] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8839-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:24.404538] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8839-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.439791] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C685D-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.459486] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:24.459502] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:24.459508] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:24.460787] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:24.499543] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0A4-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.499561] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E713-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.504442] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF20D-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.520721] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A6D-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82226562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}) [2025-08-15 10:15:24.520761] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A6D-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2025-08-15 10:15:24.531601] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:24.531613] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:24.531619] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:24.532186] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:24.532203] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:24.532211] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:24.532945] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:24.537411] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224122804206004, replayable_scn:1755224122804206004, readable_scn:1755224122804206004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224124534379, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224122804206004, replayable_scn:1755224122804206004, readable_scn:1755224122804206004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224124534379, ora_rowscn_=1755224124291763002, content_changed=true) [2025-08-15 10:15:24.537576] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:24.537588] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:24.537594] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:24.543635] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:24.543645] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:24.543653] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:24.548921] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:104) [17458][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=12] start checking lost replicas [2025-08-15 10:15:24.548946] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:24.548962] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:24.548973] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:24.550336] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:24.602236] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B0-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:24.604748] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B0-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:24.619205] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:24.620447] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224124619131, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:15:24.620473] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224129620471, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:15:24.643867] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C54-0-0] [lt=12] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1011, thread_idx=0) [2025-08-15 10:15:24.645103] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1011, only_for_display=false) [2025-08-15 10:15:24.646299] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:24.660003] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.661164] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.662306] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.662330] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1011, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:24.662345] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.662354] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1011, ls_id:{id:1}, replicas:[{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:24.662457] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=80] build ls locality stat map(tenant_id=1011, ls_id={id:1}) [2025-08-15 10:15:24.662465] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.662476] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.662488] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.662499] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:24.662574] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.662587] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.662594] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.662600] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.662607] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3789, tenant_id=1011, ls_id={id:1}) [2025-08-15 10:15:24.662616] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:24.663689] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18587, tenant_id=1011, acc_dr_task=0) [2025-08-15 10:15:24.664906] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1012, only_for_display=false) [2025-08-15 10:15:24.666096] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:24.678895] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9592-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.679740] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.680930] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.681429] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9592-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:24.681444] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9592-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.682104] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.682125] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:24.682140] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.682149] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1}, replicas:[{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:24.682250] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=79] build ls locality stat map(tenant_id=1012, ls_id={id:1}) [2025-08-15 10:15:24.682260] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.682272] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.682280] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.682289] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:24.682365] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.682377] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.682384] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.682392] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.682399] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3798, tenant_id=1012, ls_id={id:1}) [2025-08-15 10:15:24.682406] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:24.683491] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:24.688512] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6860-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.698121] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885E-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.699792] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD72-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.701900] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885E-0-0] [lt=86] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:24.701922] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885E-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.705607] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:24.705618] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:24.705623] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:24.706656] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.706899] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:24.707799] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.708935] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.708964] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:24.708982] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.708993] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1001}, replicas:[{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:24.709093] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=79] build ls locality stat map(tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:15:24.709101] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.709113] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.709125] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.709136] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:24.709214] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.709228] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.709238] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.709244] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.709253] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3720, tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:15:24.709265] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:24.710310] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:24.710795] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8360-0-0] [lt=5] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224124305449004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224124305449004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224124305449004, v:0}}], last_dump_ts_=1755224119680063, force_dump=false) [2025-08-15 10:15:24.710819] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8360-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.713915] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8360-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.715838] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8360-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.715935] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8360-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:24.725231] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D2-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:24.725897] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB25-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.726809] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5247-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:24.727779] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D2-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:24.728474] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB25-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:24.728491] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB25-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:24.729297] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5247-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:24.730873] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.732034] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.733177] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.733199] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:24.733216] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.733227] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1002}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:24.733333] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=82] build ls locality stat map(tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:15:24.733341] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.733354] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.733365] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.733377] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:24.733454] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.733464] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.733474] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.733481] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.733489] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3749, tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:15:24.733500] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:24.734561] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:24.749022] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF04B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.749047] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD81-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.755028] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.756176] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.757323] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:24.757346] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:24.757363] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.757374] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1003}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:24.757476] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=79] build ls locality stat map(tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:15:24.757485] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.757498] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.757511] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=10] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:24.757522] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:24.757598] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.757616] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=10] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.757627] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.757637] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:24.757647] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3762, tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:15:24.757659] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:24.758728] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=95025, tenant_id=1012, acc_dr_task=0) [2025-08-15 10:15:24.798042] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB867D-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:24.800511] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB867D-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:24.802277] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E6-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:24.804757] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:24.846741] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D01-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.883825] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B35B-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:24.889106] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8C-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:24.891672] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8C-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:24.905113] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224124534379, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224122804206004, replayable_scn:1755224122804206004, readable_scn:1755224122804206004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:24.928500] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6290-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:24.931034] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6290-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:24.986215] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED2-0-0] [lt=12] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1007, thread_idx=0) [2025-08-15 10:15:24.987482] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1007, only_for_display=false) [2025-08-15 10:15:24.988719] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=17] start try ls disaster recovery(ls_status_info={tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:24.990072] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58CC-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.990778] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9AF-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:24.995850] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17739][T1_TimerWK2_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:25.002639] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.003795] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.004947] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.004971] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1007, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:25.004987] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.004999] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1007, ls_id:{id:1}, replicas:[{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:25.005103] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=80] build ls locality stat map(tenant_id=1007, ls_id={id:1}) [2025-08-15 10:15:25.005110] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.005124] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.005134] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.005147] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:25.005225] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.005239] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.005249] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.005257] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.005266] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3786, tenant_id=1007, ls_id={id:1}) [2025-08-15 10:15:25.005276] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:25.006343] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18862, tenant_id=1007, acc_dr_task=0) [2025-08-15 10:15:25.007601] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1008, only_for_display=false) [2025-08-15 10:15:25.008804] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:25.022922] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.024071] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.025214] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.025237] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:25.025254] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.025264] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1}, replicas:[{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:25.025364] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=78] build ls locality stat map(tenant_id=1008, ls_id={id:1}) [2025-08-15 10:15:25.025374] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.025386] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.025394] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.025404] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:25.025479] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.025488] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.025496] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.025503] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.025510] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3750, tenant_id=1008, ls_id={id:1}) [2025-08-15 10:15:25.025518] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:25.026565] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:25.047509] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.048702] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.049869] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.049889] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:25.049904] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.049913] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1001}, replicas:[{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:25.050020] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=84] build ls locality stat map(tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:15:25.050027] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.050038] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.050046] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.050056] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:25.050131] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.050143] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.050150] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.050158] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.050165] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3792, tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:15:25.050174] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:25.051252] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:25.071975] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.073158] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.074310] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.074332] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:25.074351] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.074360] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1002}, replicas:[{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:25.074460] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=78] build ls locality stat map(tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:15:25.074468] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.074483] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=12] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.074492] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.074501] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:25.074575] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.074587] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.074594] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.074601] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.074608] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3784, tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:15:25.074617] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:25.075663] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:25.096433] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.097590] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.098758] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:25.098781] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:25.098797] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.098809] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1003}, replicas:[{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:25.098918] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=85] build ls locality stat map(tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:15:25.098926] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.098944] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=14] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.098951] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:25.098963] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:25.099040] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.099055] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.099064] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.099074] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:25.099080] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3802, tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:15:25.099090] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:25.100212] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93853, tenant_id=1008, acc_dr_task=0) [2025-08-15 10:15:25.102528] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58C6-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.134853] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0A7-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.147284] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF048-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.183979] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8CFF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.198612] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9B4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.214173] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FF-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.214850] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF57-0-0] [lt=43] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.216690] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FF-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:25.216706] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB99FF-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.217372] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF57-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:25.217387] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF57-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.295930] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C07-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.296020] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC039-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.296133] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D2-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.296416] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B1-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.296496] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7840-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.299161] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC039-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.299161] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C07-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.299357] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B1-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.299357] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D2-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.299533] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7840-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.301358] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B1-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.301367] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC039-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.301410] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D2-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.301451] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B1-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.301480] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC039-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.301504] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D2-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.301716] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C07-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.301807] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C07-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.302079] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7840-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.302245] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7840-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.358598] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7630-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.361746] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7630-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.363669] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7630-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.363754] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7630-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.406830] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A19-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.407344] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883A-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.409378] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A19-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:25.409393] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A19-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.409977] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883A-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:25.409996] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883A-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.439819] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C685E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.459627] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:25.459639] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:25.459644] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:25.460876] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A06FB-0-0] [lt=13] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.93164062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}) [2025-08-15 10:15:25.460917] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A06FB-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:25.499618] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0A5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.499635] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E714-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.531431] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224124352755000, replayable_scn:1755224124352755000, readable_scn:1755224124352755000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224125527854, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224124352755000, replayable_scn:1755224124352755000, readable_scn:1755224124352755000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224125527854, ora_rowscn_=1755224125360613000, content_changed=true) [2025-08-15 10:15:25.531621] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:25.531634] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:25.531640] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:25.532293] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:25.532306] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:25.532314] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:25.537637] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:25.537648] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:25.537653] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:25.543730] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:25.543741] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:25.543747] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:25.549084] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:25.549095] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:25.549103] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:25.610067] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B1-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:25.612544] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B1-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:25.614053] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:25.615318] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224125613976, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:15:25.615343] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224130615341, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:15:25.668154] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438023][T1006_TimerWK2_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:25.678785] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC634-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.684176] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9593-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.686786] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9593-0-0] [lt=76] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:25.686805] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9593-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.688587] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6861-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.699873] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD73-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.704679] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885F-0-0] [lt=45] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.705701] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:25.705712] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:25.705720] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:25.707223] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885F-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:25.707242] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD885F-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.716970] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8361-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.720095] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8361-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.722016] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8361-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.722204] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8361-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:25.731211] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB26-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.733158] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D3-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:25.733730] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB26-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:25.733747] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB26-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:25.734559] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5248-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:25.735670] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D3-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:25.737044] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5248-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:25.749096] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF04C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.749118] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD82-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.805843] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB867E-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:25.808411] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB867E-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:25.810131] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E7-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:25.812705] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E7-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:25.846810] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D02-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.885238] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B35C-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:25.897139] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8D-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:25.899609] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8D-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:25.936379] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6291-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:25.938734] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:25.938895] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6291-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:25.939974] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224125938650, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:15:25.940000] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224130939998, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:15:25.954822] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:25.956028] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224125946159, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:15:25.956049] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224130956047, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:15:25.956064] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224130956061, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:25.990150] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58CD-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:25.990853] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9B0-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.102602] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58C7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.134927] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0A8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.147331] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF049-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.181587] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=6] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:26.182829] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224126181515, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:15:26.182854] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224131182852, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:15:26.183764] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:26.184052] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D00-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.185548] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224126183698, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:15:26.185574] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224131185573, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:26.198686] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9B5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.219412] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A00-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.220050] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF58-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.221967] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A00-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:26.221987] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A00-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.222574] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF58-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:26.222592] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF58-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.271843] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BB9-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}) [2025-08-15 10:15:26.271888] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BB9-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=50, ddl_arg=NULL) [2025-08-15 10:15:26.302439] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03A-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.302468] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D3-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.302469] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B2-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224125807580003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224125807580003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224125807580003, v:0}}], last_dump_ts_=1755224121270734, force_dump=false) [2025-08-15 10:15:26.302493] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B2-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.302817] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C08-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.303171] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7841-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.305535] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D3-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.305613] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B2-0-0] [lt=15] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224125807580003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224125807580004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224125807580004, v:0}}], last_dump_ts_=1755224121273853, force_dump=false) [2025-08-15 10:15:26.305639] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B2-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.305815] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C08-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.305856] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03A-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224125807580004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224125807580004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224125807580004, v:0}}], last_dump_ts_=1755224121273975, force_dump=false) [2025-08-15 10:15:26.305877] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03A-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.306125] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7841-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.307537] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D3-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.307637] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D3-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.307842] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C08-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.307850] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B2-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.307916] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03A-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.307950] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B2-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.308063] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C08-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.308110] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7841-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.308117] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.308190] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7841-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.364711] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7631-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.367790] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7631-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.369779] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7631-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.369888] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7631-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.412094] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1A-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.412682] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883B-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.414650] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1A-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:26.414670] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1A-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.415232] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883B-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:26.415250] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.439898] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C685F-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.459395] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224124305449004, replayable_scn:1755224124305449004, readable_scn:1755224124305449004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224126456548, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224124305449004, replayable_scn:1755224124305449004, readable_scn:1755224124305449004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224126456548, ora_rowscn_=1755224125719220000, content_changed=true) [2025-08-15 10:15:26.459582] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:26.459592] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:26.459600] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:26.499686] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E715-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.499693] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0A6-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.504750] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF20E-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.521199] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A6F-0-0] [lt=12] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83007812GB}) [2025-08-15 10:15:26.521239] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A6F-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:15:26.531693] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:26.531705] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:26.531711] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:26.531721] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1016, cost_time_us=194, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:26.532136] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224124857956003, replayable_scn:1755224124857956003, readable_scn:1755224124857956003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224126529158, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224124857956003, replayable_scn:1755224124857956003, readable_scn:1755224124857956003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224126529158, ora_rowscn_=1755224126303984000, content_changed=true) [2025-08-15 10:15:26.532299] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:26.532312] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:26.532317] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:26.537639] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:26.537650] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:26.537656] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:26.543650] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224124806491004, replayable_scn:1755224124806491004, readable_scn:1755224124806491004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224126540748, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224124806491004, replayable_scn:1755224124806491004, readable_scn:1755224124806491004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224126540748, ora_rowscn_=1755224126305038000, content_changed=true) [2025-08-15 10:15:26.543846] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:26.543859] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:26.543864] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:26.548901] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224124806491004, replayable_scn:1755224124806491004, readable_scn:1755224124806491004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224126546002, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224124806491004, replayable_scn:1755224124806491004, readable_scn:1755224124806491004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224126546002, ora_rowscn_=1755224126303984000, content_changed=true) [2025-08-15 10:15:26.549066] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:26.549077] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:26.549083] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:26.558385] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487710][T1003_DRService][T1003][YB420ACB0122-0006337FE6577D84-0-0] [lt=13] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1003, thread_idx=1) [2025-08-15 10:15:26.562109] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [1487710][T1003_DRService][T1003][YB420ACB0122-0006337FE6577D85-0-0] [lt=12] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2392, tenant_id=1003, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:26.564422] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [1487710][T1003_DRService][T1003][YB420ACB0122-0006337FE6577D85-0-0] [lt=14] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2295, tenant_id=1004, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:26.617920] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B2-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:26.620415] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B2-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:26.688659] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6862-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.689521] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9594-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.692095] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9594-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:26.692114] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9594-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.699949] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD74-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.705612] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224125203960000, replayable_scn:1755224125203960000, readable_scn:1755224125203960000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224126702656, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224125203960000, replayable_scn:1755224125203960000, readable_scn:1755224125203960000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224126702656, ora_rowscn_=1755224126305038000, content_changed=true) [2025-08-15 10:15:26.705792] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:26.705802] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:26.705813] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:26.709922] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8860-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.712447] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8860-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:26.712465] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8860-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.713305] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAAE-0-0] [lt=13] [RESTORE] restore scheduler quit [2025-08-15 10:15:26.714538] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAAE-0-0] [lt=13] get targets(ret=0, jobs=[], exec_tenant_id=1017, sql=select * from __all_recover_table_job) [2025-08-15 10:15:26.714569] INFO [SQL] alloc_value (ob_sql_session_mgr.cpp:260) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAAF-0-0] [lt=15] alloc_session_count(alloc_total_count=5601730000) [2025-08-15 10:15:26.715727] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAAF-0-0] [lt=10] get targets(ret=0, jobs=[], exec_tenant_id=1017, sql=select * from __all_import_table_job) [2025-08-15 10:15:26.715751] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAB0-0-0] [lt=15][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224126715746, ref_timestamp=1755224066711893, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:15:26.715841] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAB1-0-0] [lt=12] [RESTORE] restore scheduler start [2025-08-15 10:15:26.715856] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAB2-0-0] [lt=12] [RESTORE] try process restore job [2025-08-15 10:15:26.717034] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [728056][T1017_REST_SER][T1017][YB420ACB0122-000633A35567EAB2-0-0] [lt=10] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:15:26.723271] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8362-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.726345] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8362-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.728259] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8362-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.728350] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8362-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:26.736494] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB27-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.739109] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB27-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:26.739126] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB27-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:26.740998] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D4-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:26.742285] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5249-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:26.743493] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D4-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:26.744739] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5249-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:26.749169] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF04D-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.749192] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD83-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.770972] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:26.772830] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=69] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:26.813828] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB867F-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:26.817476] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB867F-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:26.817903] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E8-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:26.820407] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E8-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:26.823174] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:26.824387] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224126823101, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:15:26.824412] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224131824410, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:15:26.846879] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D03-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.886615] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B35D-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:26.904921] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8E-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:26.907395] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8E-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:26.944272] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6292-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:26.946766] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6292-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:26.990226] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58CE-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:26.990930] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9B1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.102674] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58C8-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.110735] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727135][T1016_TimerWK3_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:27.135005] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0A9-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.147407] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF04A-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.184126] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D01-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.198662] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:27.198758] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9B6-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.224862] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A01-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.225241] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF59-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.227403] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A01-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:27.227418] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A01-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.227773] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF59-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:27.227789] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF59-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.251599] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:27.251614] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:27.253028] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619488][T1012_TimerWK3_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=5] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:15:27.257517] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:27.258685] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:27.293620] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [17709][T1_REST_SER][T1][YB420ACB0122-0006337F03BA32B8-0-0] [lt=12] [RESTORE] restore scheduler quit [2025-08-15 10:15:27.294815] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [17709][T1_REST_SER][T1][YB420ACB0122-0006337F03BA32B8-0-0] [lt=11] get targets(ret=0, jobs=[], exec_tenant_id=1, sql=select * from __all_recover_table_job) [2025-08-15 10:15:27.294840] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [17709][T1_REST_SER][T1][YB420ACB0122-0006337F03BA32B9-0-0] [lt=14][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:27.294923] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [17709][T1_REST_SER][T1][YB420ACB0122-0006337F03BA32BA-0-0] [lt=10] [RESTORE] restore scheduler start [2025-08-15 10:15:27.294931] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [17709][T1_REST_SER][T1][YB420ACB0122-0006337F03BA32BB-0-0] [lt=6] [RESTORE] try process restore job [2025-08-15 10:15:27.296062] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [17709][T1_REST_SER][T1][YB420ACB0122-0006337F03BA32BB-0-0] [lt=5] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:15:27.308647] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D4-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.308931] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B3-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.309053] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03B-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224126808922005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224126808922005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224126808922005, v:0}}], last_dump_ts_=1755224122277081, force_dump=false) [2025-08-15 10:15:27.309061] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C09-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.309081] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03B-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.309152] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7842-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.311734] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D4-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.312043] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03B-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.312181] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7842-0-0] [lt=8] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224126808922005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224126808922005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224126808922005, v:0}}], last_dump_ts_=1755224122280877, force_dump=false) [2025-08-15 10:15:27.312201] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7842-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.312777] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B3-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.312793] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C09-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.313001] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:27.313858] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D4-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.313975] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D4-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.314031] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.314142] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03B-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.314163] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7842-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.314247] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7842-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.315416] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B3-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.315432] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C09-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.315503] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B3-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.315525] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C09-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.327810] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:27.329080] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224127327732, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:15:27.329104] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224132329103, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:15:27.370848] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7632-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.374284] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7632-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.376244] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7632-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.376350] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7632-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.414157] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:27.415394] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224127414079, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:15:27.415418] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224132415417, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:15:27.416133] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E5E-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224127415483000, start_id=1755224127415483000, end_id=1755224127415483001) [2025-08-15 10:15:27.416856] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E5E-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224127415483000, v:0}, tenant_id=1017) [2025-08-15 10:15:27.417818] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1B-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.418042] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883C-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.420690] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883C-0-0] [lt=72] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:27.420706] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883C-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.421629] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1B-0-0] [lt=93] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:27.421652] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1B-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.439972] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6860-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.459662] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:27.459677] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:27.459682] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:27.461377] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E3AF-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}) [2025-08-15 10:15:27.461422] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E3AF-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=50, ddl_arg=NULL) [2025-08-15 10:15:27.499723] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E716-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.499765] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0A7-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.531735] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:27.531746] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:27.531751] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:27.532253] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:27.532270] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:27.532279] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:27.533616] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:27.537541] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224126239831000, replayable_scn:1755224126239831000, readable_scn:1755224126239831000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224127535358, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224126239831000, replayable_scn:1755224126239831000, readable_scn:1755224126239831000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224127535358, ora_rowscn_=1755224127310439000, content_changed=true) [2025-08-15 10:15:27.537705] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:27.537724] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:27.537733] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:27.543838] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:27.543850] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:27.543858] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:27.545160] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:27.549088] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:27.549099] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:27.549104] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:27.550868] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E67-0-0] [lt=43] get number(ret=0, service_type_=0, range=1, base_id=1755224127550093000, start_id=1755224127550093000, end_id=1755224127550093001) [2025-08-15 10:15:27.551571] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E67-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224127550093000, v:0}, tenant_id=1015) [2025-08-15 10:15:27.551858] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA669-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224127551147000, start_id=1755224127551147000, end_id=1755224127551147001) [2025-08-15 10:15:27.552585] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA669-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224127551147000, v:0}, tenant_id=1005) [2025-08-15 10:15:27.625731] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B3-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:27.628237] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B3-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:27.684837] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC635-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.688732] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6863-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.695069] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9595-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.697988] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9595-0-0] [lt=95] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:27.698003] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9595-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.700024] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD75-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.705840] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:27.705854] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:27.705860] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:27.715169] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8861-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.717704] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8861-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:27.717722] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8861-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.729361] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8363-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.732309] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8363-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.734826] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8363-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.735029] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8363-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:27.741872] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB28-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.744399] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB28-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:27.744415] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB28-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:27.748773] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D5-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:27.749226] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF04E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.749264] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD84-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.749908] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD524A-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:27.752366] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD524A-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:27.752423] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D5-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:27.780432] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36C7-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224127779397000, start_id=1755224127779397000, end_id=1755224127779397001) [2025-08-15 10:15:27.781137] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36C7-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224127779397000, v:0}, tenant_id=1011) [2025-08-15 10:15:27.792551] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:27.793792] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224127792479, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:15:27.793814] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224132793813, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:15:27.794182] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:27.795349] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224127794116, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:15:27.795369] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224132795368, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:15:27.822826] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8680-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:27.825354] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8680-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:27.825691] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E9-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:27.828177] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52E9-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:27.846953] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D04-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.852684] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=7] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:27.852722] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=26] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:27.852737] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:27.853864] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=9] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:27.853904] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=28] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:27.853921] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=13] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:27.855194] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [1487604][T1004_TimerWK1_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=16] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:27.855196] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=9] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:27.888039] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B35E-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:27.898528] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FAF-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224127898225000, start_id=1755224127898225000, end_id=1755224127898225001) [2025-08-15 10:15:27.899193] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FAF-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224127898225000, v:0}, tenant_id=1013) [2025-08-15 10:15:27.906633] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=12] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224127535358, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224126239831000, replayable_scn:1755224126239831000, readable_scn:1755224126239831000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:27.912581] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8F-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:27.915105] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F8F-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:27.952058] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6293-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:27.954540] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6293-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:27.955974] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:27.957208] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224127955899, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:15:27.957233] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224132957231, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:15:27.990298] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58CF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:27.991008] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9B2-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.017446] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826E-0-0] [lt=12] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1005, thread_idx=0) [2025-08-15 10:15:28.018726] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1005, only_for_display=false) [2025-08-15 10:15:28.019968] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:28.033726] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.034887] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.036036] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.036057] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1005, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:28.036071] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.036081] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1005, ls_id:{id:1}, replicas:[{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:28.036184] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=81] build ls locality stat map(tenant_id=1005, ls_id={id:1}) [2025-08-15 10:15:28.036192] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.036203] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.036212] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.036221] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:28.036303] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.036315] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.036322] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.036329] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.036334] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3766, tenant_id=1005, ls_id={id:1}) [2025-08-15 10:15:28.036341] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:28.037422] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18696, tenant_id=1005, acc_dr_task=0) [2025-08-15 10:15:28.038624] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1006, only_for_display=false) [2025-08-15 10:15:28.039871] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:28.041761] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:28.043012] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224128041686, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:15:28.043037] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224133043035, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:15:28.053536] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.054703] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.055840] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.055860] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:28.055872] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.055881] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1}, replicas:[{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:28.055987] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=83] build ls locality stat map(tenant_id=1006, ls_id={id:1}) [2025-08-15 10:15:28.055997] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.056009] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.056017] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.056027] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:28.056113] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=64] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.056122] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=3] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.056129] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.056137] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.056143] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3747, tenant_id=1006, ls_id={id:1}) [2025-08-15 10:15:28.056151] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:28.057225] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:28.075808] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36AF-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224128074958000, start_id=1755224128074958000, end_id=1755224128074958001) [2025-08-15 10:15:28.076686] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36AF-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224128074958000, v:0}, tenant_id=1012) [2025-08-15 10:15:28.077959] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.079157] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.080312] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.080334] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:28.080347] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.080356] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1001}, replicas:[{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:28.080455] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=78] build ls locality stat map(tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:15:28.080465] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.080477] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.080485] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.080494] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:28.080570] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.080579] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.080587] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.080594] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.080601] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3797, tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:15:28.080609] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:28.081665] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:28.096386] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463B7-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224128096022000, start_id=1755224128096022000, end_id=1755224128096022001) [2025-08-15 10:15:28.097144] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463B7-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224128096022000, v:0}, tenant_id=1) [2025-08-15 10:15:28.102355] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.102748] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58C9-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.103485] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.104642] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.104663] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:28.104678] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.104687] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1002}, replicas:[{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:28.104788] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=78] build ls locality stat map(tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:15:28.104796] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.104806] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.104814] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.104824] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:28.104920] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=72] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.104932] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.104945] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.104952] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.104959] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3718, tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:15:28.104967] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:28.106059] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:28.126696] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.127847] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.128997] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.129018] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:28.129030] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.129041] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1003}, replicas:[{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:28.129149] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=84] build ls locality stat map(tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:15:28.129159] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.129171] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.129180] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.129189] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:28.129263] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=54] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.129276] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.129284] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.129291] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.129298] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3734, tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:15:28.129307] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:28.130383] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=92947, tenant_id=1006, acc_dr_task=0) [2025-08-15 10:15:28.135077] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0AA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.147480] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF04B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.184200] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D02-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.198830] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9B7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.230194] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A02-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.230523] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5A-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.232718] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A02-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:28.232735] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A02-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.233086] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5A-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:28.233101] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5A-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.272319] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9F9EE-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}) [2025-08-15 10:15:28.272363] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9F9EE-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:15:28.315005] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D5-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224127810896004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224127810896004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224127810896004, v:0}}], last_dump_ts_=1755224123283050, force_dump=false) [2025-08-15 10:15:28.315034] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D5-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.315122] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.315267] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7843-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.316504] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.316522] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B4-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.318208] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03C-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.318310] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D5-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224127810896004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224127810896004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224127810896004, v:0}}], last_dump_ts_=1755224123286260, force_dump=false) [2025-08-15 10:15:28.318335] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D5-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.318369] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7843-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.319510] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B4-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.319556] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.320288] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.320345] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7843-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.320373] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.320443] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7843-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.320462] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D5-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.320543] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D5-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.321535] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.321599] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B4-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.321623] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.321683] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B4-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.377301] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7633-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.380263] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7633-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.382240] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7633-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.382347] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7633-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.423464] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883D-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.424361] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1C-0-0] [lt=49] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.426119] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883D-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:28.426137] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.426875] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1C-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:28.426893] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.440046] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6861-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.459731] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:28.459744] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:28.459750] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:28.499798] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E717-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.499837] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0A8-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.505034] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF20F-0-0] [lt=10] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.521682] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1277-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83007812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83398438GB}) [2025-08-15 10:15:28.521726] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1277-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=47, ddl_arg=NULL) [2025-08-15 10:15:28.531757] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=12] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224127216863000, replayable_scn:1755224127216863000, readable_scn:1755224127216863000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224128528442, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224127216863000, replayable_scn:1755224127216863000, readable_scn:1755224127216863000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224128528442, ora_rowscn_=1755224128379109000, content_changed=true) [2025-08-15 10:15:28.531990] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=25] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:28.532009] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:28.532022] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:28.532368] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:28.532379] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:28.532387] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:28.537835] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:28.537846] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:28.537851] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:28.539146] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:28.543891] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:28.543902] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:28.543910] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:28.549160] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:28.549179] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:28.549185] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:28.572040] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DED-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1, thread_idx=0) [2025-08-15 10:15:28.573310] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-08-15 10:15:28.574588] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:28.588401] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.589558] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.590708] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:28.590735] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:28.590752] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.590763] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:28.590871] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=84] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-08-15 10:15:28.590879] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.590892] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.590903] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:28.590914] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:28.590995] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.591010] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.591020] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.591029] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:28.591036] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3783, tenant_id=1, ls_id={id:1}) [2025-08-15 10:15:28.591047] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:28.592122] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18813, tenant_id=1, acc_dr_task=0) [2025-08-15 10:15:28.599692] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:28.600919] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224128599620, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:15:28.600953] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=25] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224133600951, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:15:28.633600] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B4-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:28.636082] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B4-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:28.688808] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6864-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.700094] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD76-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.700750] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9596-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.703315] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9596-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:28.703332] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9596-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.705851] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:28.705866] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:28.705874] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:28.720411] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8862-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.722946] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8862-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:28.722963] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8862-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.733618] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E57-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224128733606000, start_id=1755224128733606000, end_id=1755224128733606001) [2025-08-15 10:15:28.733945] INFO [LIB] stat (utility.h:1211) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E57-0-0] [lt=15] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=19, this=0x7f15ded8d470, extra_info="") [2025-08-15 10:15:28.733968] INFO [LIB] stat (utility.h:1211) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E57-0-0] [lt=14] [PALF STAT APPEND COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x7f15ded48050, extra_info="id:1") [2025-08-15 10:15:28.734354] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E57-0-0] [lt=8] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224128733606000, v:0}, tenant_id=1016) [2025-08-15 10:15:28.735994] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8364-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.738972] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8364-0-0] [lt=23] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224128310763004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224128310763004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224128310763004, v:0}}], last_dump_ts_=1755224123707648, force_dump=false) [2025-08-15 10:15:28.739004] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8364-0-0] [lt=25] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.740900] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8364-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.741028] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8364-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:28.747148] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB29-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.749264] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF04F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.749336] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD85-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.749677] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB29-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:28.749693] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB29-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:28.757659] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D6-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:28.757663] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD524B-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:28.760160] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:28.760202] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD524B-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:28.761863] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [727959][T1018_TimerWK3_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=14] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:28.761897] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [727959][T1018_TimerWK3_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=23] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:28.761913] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [727959][T1018_TimerWK3_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=12] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:28.763130] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [727957][T1018_TimerWK1_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=13] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:28.830693] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8681-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:28.833276] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8681-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:28.833427] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52EA-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:28.835869] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52EA-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:28.847026] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D05-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.889471] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B35F-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:28.920378] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F90-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:28.922855] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F90-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:28.959800] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6294-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:28.962297] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6294-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:28.985946] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E60-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1003, thread_idx=0) [2025-08-15 10:15:28.987242] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1003, only_for_display=false) [2025-08-15 10:15:28.988476] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:28.990376] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58D0-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:28.991082] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9B3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.002419] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.003578] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.004753] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.004777] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1003, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:29.004790] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.004800] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1003, ls_id:{id:1}, replicas:[{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:29.004913] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=89] build ls locality stat map(tenant_id=1003, ls_id={id:1}) [2025-08-15 10:15:29.004922] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.004935] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.004948] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.004959] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:29.005043] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.005057] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.005067] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.005073] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.005082] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3829, tenant_id=1003, ls_id={id:1}) [2025-08-15 10:15:29.005093] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:29.006207] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18966, tenant_id=1003, acc_dr_task=0) [2025-08-15 10:15:29.007426] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1004, only_for_display=false) [2025-08-15 10:15:29.008635] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:29.022424] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.023620] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.024780] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.024803] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:29.024817] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.024829] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:29.024932] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=81] build ls locality stat map(tenant_id=1004, ls_id={id:1}) [2025-08-15 10:15:29.024945] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.024958] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.024969] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.024984] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:29.025068] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.025078] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.025088] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.025094] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.025105] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=9] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3858, tenant_id=1004, ls_id={id:1}) [2025-08-15 10:15:29.025116] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:29.026185] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:29.047109] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.048257] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.049399] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.049421] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:29.049438] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.049449] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:29.049552] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=80] build ls locality stat map(tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:15:29.049561] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.049574] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.049585] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.049596] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:29.049671] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.049685] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.049694] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.049699] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.049707] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3738, tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:15:29.049718] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:29.050871] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:29.071664] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.072847] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.074013] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.074035] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:29.074053] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.074065] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1002}, replicas:[{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:29.074170] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=82] build ls locality stat map(tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:15:29.074178] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.074196] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.074207] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.074218] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:29.074296] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.074310] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.074319] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.074325] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.074334] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3813, tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:15:29.074348] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=10] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:29.075481] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:29.096179] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.097342] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.098509] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.098534] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:29.098550] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.098561] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1003}, replicas:[{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:29.098664] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=81] build ls locality stat map(tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:15:29.098673] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.098688] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.098702] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=11] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.098714] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:29.098792] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.098807] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.098817] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.098822] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.098832] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3792, tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:15:29.098839] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:29.099959] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93735, tenant_id=1004, acc_dr_task=0) [2025-08-15 10:15:29.102821] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58CA-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.135109] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0AB-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.147025] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64A-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224129146894000, start_id=1755224129146894000, end_id=1755224129146894001) [2025-08-15 10:15:29.147553] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF04C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.147802] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64A-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224129146894000, v:0}, tenant_id=1006) [2025-08-15 10:15:29.184269] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D03-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.198902] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9B8-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.211951] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A3-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224129211052000, start_id=1755224129211052000, end_id=1755224129211052001) [2025-08-15 10:15:29.212713] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A3-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224129211052000, v:0}, tenant_id=1008) [2025-08-15 10:15:29.235463] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A03-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.235809] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5B-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.237999] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A03-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:29.238015] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A03-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.238387] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5B-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:29.238402] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5B-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.315512] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68B9-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1017, thread_idx=0) [2025-08-15 10:15:29.316773] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1017, only_for_display=false) [2025-08-15 10:15:29.317999] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:29.321397] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7844-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224128811551005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224128811551005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224128811551005, v:0}}], last_dump_ts_=1755224124289835, force_dump=false) [2025-08-15 10:15:29.321406] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03D-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.321419] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7844-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.321482] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D6-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.322581] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B5-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.322610] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0B-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224129243647000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224129243647000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224129243647000, v:0}}], last_dump_ts_=1755224124289616, force_dump=false) [2025-08-15 10:15:29.322641] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0B-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.324645] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7844-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.324651] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.324805] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D6-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.325740] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B5-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.325861] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0B-0-0] [lt=15] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224129212104001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224129212104001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224129212104001, v:0}}], last_dump_ts_=1755224124292764, force_dump=false) [2025-08-15 10:15:29.325893] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0B-0-0] [lt=24] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.326689] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7844-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.326813] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7844-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.326816] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.326903] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.326918] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D6-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.327049] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D6-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.327710] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B5-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.327802] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B5-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.327893] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.328019] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0B-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.332207] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.333379] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.334534] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.334556] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1017, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:29.334570] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.334580] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1017, ls_id:{id:1}, replicas:[{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:29.334690] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=86] build ls locality stat map(tenant_id=1017, ls_id={id:1}) [2025-08-15 10:15:29.334700] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.334711] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.334720] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.334728] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:29.334805] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.334817] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.334824] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.334831] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.334837] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3826, tenant_id=1017, ls_id={id:1}) [2025-08-15 10:15:29.334845] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:29.335975] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19202, tenant_id=1017, acc_dr_task=0) [2025-08-15 10:15:29.337180] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1018, only_for_display=false) [2025-08-15 10:15:29.338379] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:29.352198] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.353356] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.354531] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.354552] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:29.354568] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.354577] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1}, replicas:[{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:29.354679] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1}) [2025-08-15 10:15:29.354687] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.354697] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=7] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.354705] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.354714] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:29.354789] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.354805] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.354814] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.354821] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.354827] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3790, tenant_id=1018, ls_id={id:1}) [2025-08-15 10:15:29.354835] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:29.355893] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:29.376858] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.378036] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.379173] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.379196] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:29.379211] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.379220] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1001}, replicas:[{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:29.379322] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:15:29.379333] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.379343] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.379349] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.379358] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:29.379433] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.379445] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.379451] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.379458] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.379464] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3745, tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:15:29.379472] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:29.380579] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:29.383418] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7634-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224128811551004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224129312044001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224129312044001, v:0}}], last_dump_ts_=1755224124352513, force_dump=false) [2025-08-15 10:15:29.383441] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7634-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.386579] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7634-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224128811551004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224129312044001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224129312044001, v:0}}], last_dump_ts_=1755224124355508, force_dump=false) [2025-08-15 10:15:29.386608] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7634-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.388796] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7634-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.388895] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7634-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.401675] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.402830] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.403989] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.404012] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:29.404029] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.404042] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1002}, replicas:[{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:29.404153] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=87] build ls locality stat map(tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:15:29.404164] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.404182] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=14] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.404195] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.404205] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:29.404282] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.404312] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=17] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.404324] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.404335] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=9] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.404342] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3816, tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:15:29.404353] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:29.405429] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:29.418378] INFO [RS] prepare (ob_mview_maintenance_task.cpp:176) [1487605][T1004_TimerWK2_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=15] mview maintenance task prepare success(tenant_id_=1004, round_=116350, prepare_cost_us_=1) [2025-08-15 10:15:29.426380] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.427556] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.428715] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:29.428736] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:29.428750] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.428760] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1003}, replicas:[{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:29.428860] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=78] build ls locality stat map(tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:15:29.428867] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.428878] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.428886] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:29.428895] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:29.428978] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.428990] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.428905] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883E-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.429000] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.429007] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:29.429014] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3795, tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:15:29.429022] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:29.429569] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1D-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.430103] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94112, tenant_id=1018, acc_dr_task=0) [2025-08-15 10:15:29.431587] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883E-0-0] [lt=78] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:29.431603] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.432106] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1D-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:29.432123] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1D-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.440100] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6862-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.459513] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224127309399004, replayable_scn:1755224127309399004, readable_scn:1755224127309399004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224129456098, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224127309399004, replayable_scn:1755224127309399004, readable_scn:1755224127309399004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224129456098, ora_rowscn_=1755224128737823000, content_changed=true) [2025-08-15 10:15:29.459694] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:29.459705] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:29.459711] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:29.461000] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=14] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:29.461861] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E3B3-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}) [2025-08-15 10:15:29.461904] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E3B3-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=47, ddl_arg=NULL) [2025-08-15 10:15:29.499871] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E718-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.499907] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0A9-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.531783] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:29.531799] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:29.531805] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:29.532308] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224128076011003, replayable_scn:1755224128076011003, readable_scn:1755224128076011003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224129528730, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224128076011003, replayable_scn:1755224128076011003, readable_scn:1755224128076011003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224129528730, ora_rowscn_=1755224129324668000, content_changed=true) [2025-08-15 10:15:29.532479] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:29.532490] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:29.532495] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:29.532504] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1012, cost_time_us=173, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:29.533102] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:29.537862] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:29.537876] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:29.537881] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:29.543785] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224127809843005, replayable_scn:1755224127809843005, readable_scn:1755224127809843005, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224129540285, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224127809843005, replayable_scn:1755224127809843005, readable_scn:1755224127809843005, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224129540285, ora_rowscn_=1755224129324668000, content_changed=true) [2025-08-15 10:15:29.543957] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:29.543967] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:29.543972] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:29.549015] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224127809843005, replayable_scn:1755224127809843005, readable_scn:1755224127809843005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224129545513, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224127809843005, replayable_scn:1755224127809843005, readable_scn:1755224127809843005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224129545513, ora_rowscn_=1755224129323614000, content_changed=true) [2025-08-15 10:15:29.549186] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:29.549203] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:29.549211] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:29.550587] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:29.620507] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:29.621738] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224129620471, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:15:29.621762] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224134621760, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:15:29.641432] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B5-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:29.643896] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B5-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:29.688883] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6865-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.689698] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC636-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.700170] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD77-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.705733] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224127809843005, replayable_scn:1755224127809843005, readable_scn:1755224127809843005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224129702305, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224127809843005, replayable_scn:1755224127809843005, readable_scn:1755224127809843005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224129702305, ora_rowscn_=1755224129323614000, content_changed=true) [2025-08-15 10:15:29.705924] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:29.705936] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:29.705952] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:29.706024] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9597-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.707228] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:29.708150] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E3-0-0] [lt=59] get number(ret=0, service_type_=0, range=1, base_id=1755224129707551000, start_id=1755224129707551000, end_id=1755224129707551001) [2025-08-15 10:15:29.708555] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9597-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:29.708573] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9597-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.708794] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E3-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224129707551000, v:0}, tenant_id=1003) [2025-08-15 10:15:29.725627] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8863-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.728187] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8863-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:29.728207] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8863-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.742028] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8365-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224129312044004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224129312044004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224129312044004, v:0}}], last_dump_ts_=1755224124710794, force_dump=false) [2025-08-15 10:15:29.742053] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8365-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.745123] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8365-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.747019] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8365-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.747125] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8365-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:29.749338] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF050-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.749408] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD86-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.752435] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2A-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.754996] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2A-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:29.755016] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:29.765488] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D7-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:29.765517] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD524C-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:29.768016] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D7-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:29.768020] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD524C-0-0] [lt=9] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:29.839324] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8682-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:29.841119] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52EB-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:29.841863] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8682-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:29.843556] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52EB-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:29.847099] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D06-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.847979] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C0-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224129847440000, start_id=1755224129847440000, end_id=1755224129847440001) [2025-08-15 10:15:29.848773] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C0-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224129847440000, v:0}, tenant_id=1004) [2025-08-15 10:15:29.890863] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B360-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:29.928120] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F91-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:29.930631] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F91-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:29.967679] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6295-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:29.970205] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6295-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:29.990451] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58D1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.991155] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9B4-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:29.997173] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17740][T1_TimerWK3_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=5] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:30.102894] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58CB-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.135181] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0AC-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.147625] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF04D-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.179668] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C1-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224130178752000, start_id=1755224130178752000, end_id=1755224130178752001) [2025-08-15 10:15:30.180401] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C1-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224130178752000, v:0}, tenant_id=1007) [2025-08-15 10:15:30.184340] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D04-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.198979] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9B9-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.240772] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A04-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.241123] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5C-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.243357] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A04-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:30.243375] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A04-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.243729] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5C-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:30.243745] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5C-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.263171] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:30.263192] WDIAG [RS] get_storage_iops_and_bandwidth_limit (ob_shared_storage_net_throt_service.cpp:776) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=16][errcode=-4201] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:0}, max_iops=200, max_bandwidth=25000000, ret=-4201) [2025-08-15 10:15:30.263206] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:30.269105] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:30.270275] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=1][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:30.272840] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9F9F8-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87304688GB}) [2025-08-15 10:15:30.272879] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9F9F8-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:15:30.314888] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=16] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:30.314916] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=18] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:30.314930] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=12] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:30.314959] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=11] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:30.327836] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7845-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.327862] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.327966] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D7-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.328774] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B6-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.328972] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.330993] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7845-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.331078] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D7-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.331095] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03E-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.331766] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B6-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.332032] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.332967] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7845-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.333096] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.333163] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7845-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.333174] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D7-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.333186] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.333262] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D7-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.333744] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B6-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.333859] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B6-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.333950] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.334070] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.389934] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7635-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.393061] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7635-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.394988] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7635-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.395109] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7635-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.434372] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883F-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.434769] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1E-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.436887] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883F-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:30.436904] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D883F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.437288] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1E-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:30.437306] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1E-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.440182] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6863-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.459795] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:30.459813] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:30.459822] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:30.499945] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E719-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.499980] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0AA-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.505326] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF210-0-0] [lt=10] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.522174] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A72-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83398438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83789062GB}) [2025-08-15 10:15:30.522217] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A72-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:15:30.531874] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:30.531886] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:30.531891] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:30.532515] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:30.532528] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:30.532535] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:30.538289] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224128811551005, replayable_scn:1755224128811551005, readable_scn:1755224128811551005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224130535376, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224128811551005, replayable_scn:1755224128811551005, readable_scn:1755224128811551005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224130535376, ora_rowscn_=1755224130330241000, content_changed=true) [2025-08-15 10:15:30.538455] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:30.538464] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:30.538472] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:30.543994] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:30.544005] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:30.544013] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:30.549345] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:30.549358] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:30.549364] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:30.615418] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:30.616661] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224130615341, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:15:30.616687] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224135616684, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:15:30.649221] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B6-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:30.652869] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B6-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:30.673612] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438024][T1006_TimerWK3_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=6] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:30.688959] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6866-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.700249] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD78-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.706058] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:30.706072] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:30.706078] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:30.711292] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9598-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.713858] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9598-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:30.713876] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9598-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.730887] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8864-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.733424] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8864-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:30.733440] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8864-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.748238] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8366-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.749411] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF051-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.749481] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD87-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.751229] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8366-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.753321] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8366-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.753406] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8366-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:30.757755] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2B-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.760281] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2B-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:30.760299] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:30.773389] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D8-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:30.773447] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD524D-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:30.775974] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D8-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:30.775976] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD524D-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:30.847154] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8683-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:30.847170] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D07-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.848774] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52EC-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:30.849647] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8683-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:30.851208] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52EC-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:30.892240] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B361-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:30.908276] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224130535376, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224128811551005, replayable_scn:1755224128811551005, readable_scn:1755224128811551005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:30.936011] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F92-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:30.938504] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F92-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:30.940072] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:30.941313] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224130939998, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:15:30.941341] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224135941339, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:15:30.964930] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:30.966162] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224130956047, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:15:30.966187] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224135966186, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:15:30.966203] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224135966200, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:30.975586] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6296-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:30.978134] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6296-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:30.990525] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58D2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:30.991232] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9B5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.102970] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58CC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.135253] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0AD-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.147699] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF04E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.159192] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [476155][T1007_MergeSche][T1007][YB420ACB0122-0006338212D2DB99-0-0] [lt=65] zone_merge_mgr is already loaded(tenant_id=1007, global_merge_info={tenant_id:1007, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194404313977, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194707257901, need_update:false}}, zone_merge_infos=[{tenant_id:1007, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194707260697, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194404343035, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1007, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194707264633, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194404359873, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1007, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194707266479, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194404364879, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194404228414000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:15:31.182935] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:31.184184] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224131182852, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:15:31.184209] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224136184207, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:15:31.184411] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D05-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.185646] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:31.186838] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224131185573, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:15:31.186859] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224136186858, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:31.199049] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9BA-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.246120] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A05-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.246420] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5D-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.248643] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A05-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:31.248661] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A05-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.248970] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5D-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:31.248987] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.267705] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D0-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1015, thread_idx=0) [2025-08-15 10:15:31.268966] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1015, only_for_display=false) [2025-08-15 10:15:31.270163] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:31.284001] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.285144] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.286300] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.286322] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1015, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:31.286342] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.286353] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1015, ls_id:{id:1}, replicas:[{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:31.286456] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=80] build ls locality stat map(tenant_id=1015, ls_id={id:1}) [2025-08-15 10:15:31.286464] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.286476] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.286488] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.286498] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:31.286582] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.286597] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.286606] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.286613] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.286621] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3775, tenant_id=1015, ls_id={id:1}) [2025-08-15 10:15:31.286631] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:31.287718] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18752, tenant_id=1015, acc_dr_task=0) [2025-08-15 10:15:31.288947] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1016, only_for_display=false) [2025-08-15 10:15:31.290139] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:31.303719] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.304868] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.306016] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.306039] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:31.306051] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.306063] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1}, replicas:[{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:31.306165] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=79] build ls locality stat map(tenant_id=1016, ls_id={id:1}) [2025-08-15 10:15:31.306172] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.306184] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.306196] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=10] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.306207] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:31.306290] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=59] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.306304] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.306313] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.306318] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.306326] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3752, tenant_id=1016, ls_id={id:1}) [2025-08-15 10:15:31.306336] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:31.307454] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:31.328664] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.329821] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.331003] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.331026] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:31.331042] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.331053] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1001}, replicas:[{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:31.331162] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=85] build ls locality stat map(tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:15:31.331169] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.331178] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=6] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.331187] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.331193] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:31.331271] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.331284] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.331295] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.331304] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.331313] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3786, tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:15:31.331322] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:31.332453] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:31.334166] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D8-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.334190] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.334231] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7846-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.334823] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B7-0-0] [lt=9] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224130814238005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224130814238005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224130814238005, v:0}}], last_dump_ts_=1755224126302468, force_dump=false) [2025-08-15 10:15:31.334850] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B7-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.335047] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.337341] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03F-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224130814238005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224130814238005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224130814238005, v:0}}], last_dump_ts_=1755224126305855, force_dump=false) [2025-08-15 10:15:31.337366] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03F-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.337390] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D8-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.337418] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7846-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.337872] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B7-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224130814238005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224130814238005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224130814238005, v:0}}], last_dump_ts_=1755224126305610, force_dump=false) [2025-08-15 10:15:31.337897] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B7-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.338050] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.339340] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03F-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.339427] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC03F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.339427] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D8-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.339546] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D8-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.339579] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7846-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.339680] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7846-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.339900] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B7-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.340003] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.340027] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B7-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.340112] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.353465] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.354621] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.355759] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.355782] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:31.355798] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.355809] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1002}, replicas:[{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:31.355917] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=84] build ls locality stat map(tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:15:31.355925] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.355937] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.355953] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=12] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.355960] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:31.356037] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.356050] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.356061] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.356066] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.356074] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3740, tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:15:31.356086] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:31.357131] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:31.377837] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.378993] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.380143] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:31.380164] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:31.380179] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.380188] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1003}, replicas:[{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:31.380294] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=83] build ls locality stat map(tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:15:31.380302] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.380312] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.380318] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:31.380327] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:31.380402] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.380414] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.380423] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.380433] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:31.380442] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3730, tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:15:31.380453] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:31.381544] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93812, tenant_id=1016, acc_dr_task=0) [2025-08-15 10:15:31.382897] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [727235][T1015_BackupDat][T1015][YB420ACB0122-000633A34EF59996-0-0] [lt=14] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:31.390392] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E4E-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224131389644000, start_id=1755224131389644000, end_id=1755224131389644001) [2025-08-15 10:15:31.391155] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E4E-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224131389644000, v:0}, tenant_id=1018) [2025-08-15 10:15:31.396076] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7636-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.399057] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7636-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.400916] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7636-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.401107] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7636-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.439701] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8840-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.439932] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1F-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.440256] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6864-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.442300] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8840-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:31.442318] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8840-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.442552] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1F-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:31.442574] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A1F-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.446921] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438128][T1005_DRService][T1005][YB420ACB0122-0006338209C70E45-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1005, thread_idx=1) [2025-08-15 10:15:31.450970] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [438128][T1005_DRService][T1005][YB420ACB0122-0006338209C70E46-0-0] [lt=17] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2743, tenant_id=1005, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:31.453436] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [438128][T1005_DRService][T1005][YB420ACB0122-0006338209C70E46-0-0] [lt=27] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2431, tenant_id=1006, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:31.459855] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:31.459868] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:31.459875] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:31.462323] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E3CD-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}) [2025-08-15 10:15:31.462364] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E3CD-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:31.499999] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E71A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.500053] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0AB-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.531883] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:31.531899] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:31.531908] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:31.532594] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:31.532604] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:31.532612] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:31.538583] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:31.538603] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:31.538609] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:31.544043] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:31.544055] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:31.544061] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:31.549405] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:31.549423] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:31.549430] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:31.658195] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B7-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:31.660681] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B7-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:31.688998] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6867-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.694586] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC637-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.700321] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD79-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.706056] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:31.706067] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:31.706076] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:31.716623] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9599-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.719191] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9599-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:31.719206] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D9599-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.736130] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8865-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.738658] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8865-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:31.738674] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8865-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.749488] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF052-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.749554] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD88-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.754461] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8367-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.757429] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8367-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.759319] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8367-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.759409] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8367-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:31.763047] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2C-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.765568] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2C-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:31.765586] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:31.774164] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:31.776809] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475908][T1008_TimerWK0_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:15:31.781249] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D9-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:31.781295] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD524E-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:31.783760] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8D9-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:31.783871] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD524E-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:31.785225] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7F9E-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224131785127000, start_id=1755224131785127000, end_id=1755224131785127001) [2025-08-15 10:15:31.786019] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7F9E-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224131785127000, v:0}, tenant_id=1014) [2025-08-15 10:15:31.824489] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:31.825719] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224131824410, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:15:31.825743] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224136825742, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:15:31.847239] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D08-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.854937] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8684-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:31.856578] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52ED-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:31.857451] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8684-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:31.859063] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52ED-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:31.893642] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B362-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:31.902071] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [476032][T1007_BackupDat][T1007][YB420ACB0122-0006338212777C1C-0-0] [lt=13] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:31.943811] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F93-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:31.946307] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F93-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:31.983425] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6297-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:31.985917] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6297-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:31.990596] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58D3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:31.991311] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9B6-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.102998] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58CD-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.116489] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727133][T1016_TimerWK1_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:32.135325] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0AE-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.147742] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF04F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.184482] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D06-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.199124] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9BB-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.199986] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727956][T1018_TimerWK0_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:32.251427] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A06-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.251686] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5E-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.254044] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A06-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:32.254062] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A06-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.254277] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5E-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:32.254294] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.254367] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=30] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:32.273333] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BBE-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87304688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}) [2025-08-15 10:15:32.273373] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063428CAC95BBE-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2025-08-15 10:15:32.314318] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487605][T1004_TimerWK2_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:32.329185] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:32.330440] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224132329103, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:15:32.330465] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224137330464, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:15:32.340425] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC040-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224131815685006, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224131815685006, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224131815685006, v:0}}], last_dump_ts_=1755224127309052, force_dump=false) [2025-08-15 10:15:32.340451] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC040-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.340584] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.340602] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7847-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.340967] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B8-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.341059] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.343531] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC040-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.343690] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D9-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.343693] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7847-0-0] [lt=17] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224131815685005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224131815685005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224131815685005, v:0}}], last_dump_ts_=1755224127312180, force_dump=false) [2025-08-15 10:15:32.343716] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7847-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.344693] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0E-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.344718] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B8-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.345675] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7847-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.345729] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC040-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.345766] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7847-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.345770] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D9-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.345843] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76D9-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.345925] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC040-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.346654] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.346762] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.346800] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B8-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.346992] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B8-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.357335] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E7-0-0] [lt=12] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1013, thread_idx=0) [2025-08-15 10:15:32.358591] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1013, only_for_display=false) [2025-08-15 10:15:32.359902] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:32.373963] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.375178] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.376367] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.376390] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1013, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:32.376406] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.376418] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1013, ls_id:{id:1}, replicas:[{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:32.376529] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=86] build ls locality stat map(tenant_id=1013, ls_id={id:1}) [2025-08-15 10:15:32.376542] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.376555] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.376562] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.376573] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:32.376650] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.376665] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.376674] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.376680] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.376691] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3900, tenant_id=1013, ls_id={id:1}) [2025-08-15 10:15:32.376698] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:32.377767] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19177, tenant_id=1013, acc_dr_task=0) [2025-08-15 10:15:32.379026] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1014, only_for_display=false) [2025-08-15 10:15:32.380263] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:32.394149] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.395309] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.396456] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.396479] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:32.396495] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.396507] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1}, replicas:[{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:32.396616] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=85] build ls locality stat map(tenant_id=1014, ls_id={id:1}) [2025-08-15 10:15:32.396624] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.396638] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.396646] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.396657] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:32.396733] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.396747] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.396757] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.396761] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.396769] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3805, tenant_id=1014, ls_id={id:1}) [2025-08-15 10:15:32.396780] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:32.397883] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:32.402111] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7637-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.404976] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7637-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.406869] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7637-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.406961] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7637-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.415500] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:32.416745] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224132415417, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:15:32.416769] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224137416767, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:15:32.418985] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.420163] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.421345] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.421368] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:32.421384] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.421395] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1001}, replicas:[{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:32.421498] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=80] build ls locality stat map(tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:15:32.421507] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.421519] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.421530] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.421541] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:32.421618] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.421632] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.421641] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.421646] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.421654] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3853, tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:15:32.421663] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:32.422739] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:32.440327] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6865-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.443782] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.444953] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.445034] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8841-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.445240] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A20-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.446153] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.446176] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:32.446194] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.446205] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1002}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:32.446308] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=81] build ls locality stat map(tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:15:32.446317] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.446330] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.446341] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.446348] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:32.446426] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.446442] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.446453] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.446459] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.446467] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3885, tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:15:32.446478] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:32.447566] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8841-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:32.447591] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:32.447590] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8841-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.447763] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A20-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:32.447779] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A20-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.459759] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224130313424004, replayable_scn:1755224130313424004, readable_scn:1755224130313424004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224132457060, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224130313424004, replayable_scn:1755224130313424004, readable_scn:1755224130313424004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224132457060, ora_rowscn_=1755224131755697000, content_changed=true) [2025-08-15 10:15:32.459958] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:32.459968] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:32.459973] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:32.468882] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.470095] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.471292] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:32.471316] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:32.471332] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.471343] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1003}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:32.471446] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=80] build ls locality stat map(tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:15:32.471454] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.471467] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.471479] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:32.471494] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:32.471575] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.471589] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.471599] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.471608] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:32.471614] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3912, tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:15:32.471627] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=10] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:32.472694] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94914, tenant_id=1014, acc_dr_task=0) [2025-08-15 10:15:32.500071] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E71B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.500123] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0AC-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.505605] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF211-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.522673] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D127A-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83789062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84179688GB}) [2025-08-15 10:15:32.522717] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D127A-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:15:32.531800] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224131315001001, replayable_scn:1755224131315001001, readable_scn:1755224131315001001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224132528576, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224131315001001, replayable_scn:1755224131315001001, readable_scn:1755224131315001001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224132528576, ora_rowscn_=1755224132403401000, content_changed=true) [2025-08-15 10:15:32.531998] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:32.532010] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:32.532019] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:32.532468] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224130814238005, replayable_scn:1755224130814238005, readable_scn:1755224130814238005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224132529629, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224130814238005, replayable_scn:1755224130814238005, readable_scn:1755224130814238005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224132529629, ora_rowscn_=1755224132343453000, content_changed=true) [2025-08-15 10:15:32.532656] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:32.532665] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:32.532673] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:32.533961] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:32.538036] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:32.538050] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:32.538056] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:32.538071] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1018, cost_time_us=174, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:32.543917] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224130993051004, replayable_scn:1755224130993051004, readable_scn:1755224130993051004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224132541225, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224130993051004, replayable_scn:1755224130993051004, readable_scn:1755224130993051004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224132541225, ora_rowscn_=1755224132343453000, content_changed=true) [2025-08-15 10:15:32.544099] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:32.544113] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:32.544120] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:32.545393] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:32.549202] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224130814238005, replayable_scn:1755224130814238005, readable_scn:1755224130814238005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224132546486, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224130814238005, replayable_scn:1755224130814238005, readable_scn:1755224130814238005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224132546486, ora_rowscn_=1755224132342399000, content_changed=true) [2025-08-15 10:15:32.549381] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:32.549392] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:32.549397] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:32.666051] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B8-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:32.668524] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B8-0-0] [lt=9] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:32.689074] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6868-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.700395] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD7A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.705910] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224130814238005, replayable_scn:1755224130814238005, readable_scn:1755224130814238005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224132703102, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224130814238005, replayable_scn:1755224130814238005, readable_scn:1755224130814238005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224132703102, ora_rowscn_=1755224132342399000, content_changed=true) [2025-08-15 10:15:32.706105] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:32.706118] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:32.706123] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:32.721927] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959A-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.724496] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959A-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:32.724513] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959A-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.741350] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8866-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.743921] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8866-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:32.743937] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8866-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.749560] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF053-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.749628] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD89-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.760396] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8368-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.763138] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:340) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8368-0-0] [lt=16] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1014, ls_id:{id:1}, sync_scn:{val:1755224132316086005, v:0}, readable_scn:{val:1755224132316086005, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-08-15 10:15:32.763485] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8368-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.765442] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8368-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.765560] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8368-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:32.765600] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:459) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8368-0-0] [lt=5] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1755224130313424004, v:0}) [2025-08-15 10:15:32.768360] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2D-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.770886] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2D-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:32.770904] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2D-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:32.774052] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [727134][T1016_TimerWK2_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=10] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:32.774088] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [727134][T1016_TimerWK2_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=24] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:32.774102] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [727134][T1016_TimerWK2_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:32.775840] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [727135][T1016_TimerWK3_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=14] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:32.789033] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DA-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:32.789094] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD524F-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:32.791553] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DA-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:32.792802] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD524F-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:32.793842] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:32.795114] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224132793813, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:15:32.795138] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224137795136, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:15:32.795439] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:32.796602] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224132795368, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:15:32.796622] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224137796621, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:15:32.847311] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D09-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.862736] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8685-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:32.864341] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52EE-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:32.865284] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8685-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:32.866849] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52EE-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:32.895045] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B363-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:32.951543] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F94-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:32.955213] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F94-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:32.957305] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:32.958508] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224132957231, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:15:32.958538] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224137958537, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:15:32.990670] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58D4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.991249] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6298-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:32.991387] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9B7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:32.993763] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6298-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:33.043106] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:33.044343] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224133043035, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:15:33.044367] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224138044365, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:15:33.103071] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58CE-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.135394] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0AF-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.147813] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF050-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.184556] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D07-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.199193] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9BC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.256830] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A07-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.257008] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5F-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.259404] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A07-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:33.259423] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A07-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.259585] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5F-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:33.259604] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF5F-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.274539] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:33.274557] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:33.280551] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:33.281760] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=1][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:33.346762] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7848-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.346790] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DA-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224132817610004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224132817610004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224132817610004, v:0}}], last_dump_ts_=1755224128315003, force_dump=false) [2025-08-15 10:15:33.346817] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DA-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.346879] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC041-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.347774] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.347906] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B9-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.349790] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7848-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.349948] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC041-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.350098] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DA-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224132817610004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224132817610004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224132817610004, v:0}}], last_dump_ts_=1755224128318309, force_dump=false) [2025-08-15 10:15:33.350123] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DA-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.350655] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0F-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.350681] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B9-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.351798] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7848-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.351893] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7848-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.352008] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC041-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.352141] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC041-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.352273] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DA-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.352370] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DA-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.352621] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B9-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.352699] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0F-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.352713] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413B9-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.352798] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C0F-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.407983] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7638-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.410911] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7638-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.412821] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7638-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.412953] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7638-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.440404] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6866-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.450359] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8842-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.450436] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A21-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.452937] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8842-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:33.452965] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A21-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:33.452960] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8842-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.452982] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A21-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.460001] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:33.460013] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:33.460018] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:33.462783] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E3CF-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}) [2025-08-15 10:15:33.462822] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E3CF-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:15:33.500148] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E71C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.500197] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0AD-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.530487] INFO [RS] prepare (ob_mview_maintenance_task.cpp:176) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=31] mview maintenance task prepare success(tenant_id_=1012, round_=77507, prepare_cost_us_=0) [2025-08-15 10:15:33.532011] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:33.532023] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:33.532032] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:33.532647] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:33.532658] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:33.532665] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:33.538089] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:33.538103] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:33.538109] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:33.539425] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:33.544152] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:33.544172] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:33.544185] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:33.549392] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:33.549405] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:33.549411] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:33.601022] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:33.602257] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224133600951, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:15:33.602274] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224138602273, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:15:33.673891] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B9-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:33.676395] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38B9-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:33.689151] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6869-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.699449] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC638-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.700471] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD7B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.706134] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:33.706146] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:33.706152] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:33.727198] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959B-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.729717] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959B-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:33.729736] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959B-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.746612] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8867-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.749121] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8867-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:33.749141] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8867-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.749628] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF054-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.749700] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD8A-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.766605] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8369-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.769579] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8369-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224133317438003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224133317438004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224133317438004, v:0}}], last_dump_ts_=1755224128738971, force_dump=false) [2025-08-15 10:15:33.769603] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8369-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.771552] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8369-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.771663] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8369-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:33.773684] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2E-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.776243] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2E-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:33.776260] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:33.796855] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DB-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:33.798066] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5250-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:33.799317] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DB-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:33.800531] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5250-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:33.847388] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D0A-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.870627] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8686-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:33.872149] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52EF-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:33.873157] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8686-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:33.875827] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52EF-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:33.896415] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B364-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:33.909895] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224133535117, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224131815685005, replayable_scn:1755224131815685005, readable_scn:1755224131815685005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:33.960552] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F95-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:33.963070] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F95-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:33.990747] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58D5-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.991464] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9B8-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:33.999112] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6299-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:34.000311] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6299-0-0] [lt=12] log restore source is empty, just skip(ret=-4018, user_tenant_id=1008) [2025-08-15 10:15:34.002820] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB6299-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:34.065228] INFO [RS.LB] idle (ob_root_balancer.cpp:87) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=30] root balance idle(idle_time=10000000) [2025-08-15 10:15:34.065242] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:70) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=10] start check unit migrate finish [2025-08-15 10:15:34.065254] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=5] try check migrate unit finish by tenant(tenant_id=1) [2025-08-15 10:15:34.074556] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=6] try check migrate unit finish by tenant(tenant_id=1004) [2025-08-15 10:15:34.100290] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=12] try check migrate unit finish by tenant(tenant_id=1006) [2025-08-15 10:15:34.103149] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58CF-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.125854] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=11] try check migrate unit finish by tenant(tenant_id=1008) [2025-08-15 10:15:34.135467] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0B0-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.147885] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF051-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.151401] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1012) [2025-08-15 10:15:34.177039] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1014) [2025-08-15 10:15:34.184627] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D08-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.199265] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9BD-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.202600] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=10] try check migrate unit finish by tenant(tenant_id=1016) [2025-08-15 10:15:34.228669] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1018) [2025-08-15 10:15:34.254310] INFO [RS] check (ob_alter_locality_finish_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DA-0-0] [lt=13] start to check alter locality finish [2025-08-15 10:15:34.255477] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DB-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.256625] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DB-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.258915] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DD-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.260061] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DD-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.262137] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A08-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.262318] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DF-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.262579] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF60-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.263457] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814DF-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.264757] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A08-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:34.264775] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A08-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.265774] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814E1-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.266211] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF60-0-0] [lt=93] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:34.266234] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF60-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.266915] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814E1-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.269209] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814E3-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.270361] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814E3-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.272639] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814E5-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.273765] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814E5-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.273841] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B6348-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}) [2025-08-15 10:15:34.273880] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B6348-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:15:34.276062] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814E7-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.277222] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814E7-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.279497] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814E9-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.280634] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814E9-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.281792] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814E9-0-0] [lt=12] start check shrink resource pool [2025-08-15 10:15:34.281810] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EA-0-0] [lt=12] start check shrink resource pool(tenant_id=1) [2025-08-15 10:15:34.282904] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EA-0-0] [lt=5][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.284041] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EA-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.284091] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EB-0-0] [lt=12] start check shrink resource pool(tenant_id=1004) [2025-08-15 10:15:34.285188] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EB-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.286327] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EB-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.286378] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EC-0-0] [lt=12] start check shrink resource pool(tenant_id=1006) [2025-08-15 10:15:34.287505] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EC-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.288639] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EC-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.288688] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814ED-0-0] [lt=11] start check shrink resource pool(tenant_id=1008) [2025-08-15 10:15:34.289795] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814ED-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.290942] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814ED-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.290992] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EE-0-0] [lt=12] start check shrink resource pool(tenant_id=1012) [2025-08-15 10:15:34.292100] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EE-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.293223] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EE-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.293271] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EF-0-0] [lt=12] start check shrink resource pool(tenant_id=1014) [2025-08-15 10:15:34.294377] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EF-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.295500] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814EF-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.295549] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F0-0-0] [lt=13] start check shrink resource pool(tenant_id=1016) [2025-08-15 10:15:34.296665] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F0-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.297821] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F0-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.297871] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F1-0-0] [lt=11] start check shrink resource pool(tenant_id=1018) [2025-08-15 10:15:34.298986] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F1-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.300152] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F1-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.300202] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:99) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F1-0-0] [lt=12] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.300215] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F1-0-0] [lt=9] start check primary zone [2025-08-15 10:15:34.300227] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F2-0-0] [lt=7] start check primary zone(tenant_id=1) [2025-08-15 10:15:34.301341] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F2-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.301389] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F3-0-0] [lt=11] start check primary zone(tenant_id=1004) [2025-08-15 10:15:34.302492] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F3-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.302541] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F4-0-0] [lt=12] start check primary zone(tenant_id=1006) [2025-08-15 10:15:34.303631] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F4-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.303679] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F5-0-0] [lt=12] start check primary zone(tenant_id=1008) [2025-08-15 10:15:34.304772] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F5-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.304819] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F6-0-0] [lt=12] start check primary zone(tenant_id=1012) [2025-08-15 10:15:34.305931] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F6-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.305988] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F7-0-0] [lt=17] start check primary zone(tenant_id=1014) [2025-08-15 10:15:34.307094] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F7-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.307144] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F8-0-0] [lt=12] start check primary zone(tenant_id=1016) [2025-08-15 10:15:34.308254] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F8-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.308302] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=12] start check primary zone(tenant_id=1018) [2025-08-15 10:15:34.309409] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:34.309460] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:166) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=13] start do unit balance [2025-08-15 10:15:34.315612] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:350) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=9] start distribute for server status change [2025-08-15 10:15:34.315712] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:401) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=11] finish distribute for server status change(ret=0) [2025-08-15 10:15:34.315723] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:200) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=8] finish do unit balance(ret=0) [2025-08-15 10:15:34.321705] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:115) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=4] start to do tenant group unit balance [2025-08-15 10:15:34.321747] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=12] start to do tenant group unit balance(zone=zone1) [2025-08-15 10:15:34.321761] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=9] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.203.1.34:2882", disk_in_use_:3105882112, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:34.321780] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=12] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:34.321809] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:34.321827] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=9] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:34.321837] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone1) [2025-08-15 10:15:34.321848] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=8] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:34.321870] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=13] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.1.34:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:34.321919] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=38] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.203.1.34:2882", disk_in_use_:3105882112, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:34.321927] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=6] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.1.34:2882", disk_in_use_:3105882112, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:34.321945] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:34.321959] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=11] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1006, unit_config:{unit_config_id:1005, name:"config_opsone_zone1_s_2_4_ywx", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7988, unit_config:0x7f1e9aa71c38, pool:0x7f1c3beb2f18}, {unit_id:1008, tenant_id:1006, resource_pool_id:1008, unit_config:{unit_config_id:1008, name:"config_test_zone1_s_2_4_emp", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f77d8, unit_config:0x7f1e9aa71998, pool:0x7f1c3beb27a8}, {unit_id:1011, tenant_id:1008, resource_pool_id:1011, unit_config:{unit_config_id:1011, name:"config_mcp_zone1_s_4_8_tmc", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7550, unit_config:0x7f1e9aa716f8, pool:0x7f1e443fda40}, {unit_id:1016, tenant_id:1012, resource_pool_id:1016, unit_config:{unit_config_id:1017, name:"config_oms_zone1_s_4_8_gzp", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f73a0, unit_config:0x7f1e9aa71458, pool:0x7f1e443fd2d0}, {unit_id:1019, tenant_id:1014, resource_pool_id:1019, unit_config:{unit_config_id:1019, name:"config_grafana_test_zone1_s_1_2_pri", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7118, unit_config:0x7f1e9aa71298, pool:0x7f1e443fc7a8}, {unit_id:1024, tenant_id:1016, resource_pool_id:1024, unit_config:{unit_config_id:1023, name:"config_temporalio_zone1_s_2_4_ykw", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6ce0, unit_config:0x7f1e9aa70f18, pool:0x7f1e74dff2d0}, {unit_id:1026, tenant_id:1018, resource_pool_id:1026, unit_config:{unit_config_id:1027, name:"config_media_zone1_s_2_4_kmn", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6b30, unit_config:0x7f1e9aa70b98, pool:0x7f1e74dfeb60}], not_grant_units=[], available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:34.322022] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=39] start to do tenant group unit balance(zone=zone2) [2025-08-15 10:15:34.322031] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.203.4.38:2882", disk_in_use_:3093299200, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:34.322044] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=9] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:34.322062] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=9] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:34.322074] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=7] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:34.322083] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone2) [2025-08-15 10:15:34.322088] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:34.322102] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.38:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:34.322134] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=26] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.203.4.38:2882", disk_in_use_:3093299200, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:34.322140] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=4] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.38:2882", disk_in_use_:3093299200, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:34.322150] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:34.322161] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=8] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1004, unit_config:{unit_config_id:1004, name:"config_opsone_zone2_s_2_4_wag", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7b38, unit_config:0x7f1e9aa71d18, pool:0x7f1c3beb3688}, {unit_id:1009, tenant_id:1006, resource_pool_id:1009, unit_config:{unit_config_id:1009, name:"config_test_zone2_s_2_4_yey", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7700, unit_config:0x7f1e9aa718b8, pool:0x7f1c3beb23f0}, {unit_id:1010, tenant_id:1008, resource_pool_id:1010, unit_config:{unit_config_id:1010, name:"config_mcp_zone2_s_4_8_bjb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7628, unit_config:0x7f1e9aa717d8, pool:0x7f1c3beb2038}, {unit_id:1018, tenant_id:1012, resource_pool_id:1018, unit_config:{unit_config_id:1016, name:"config_oms_zone2_s_4_8_kbx", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f71f0, unit_config:0x7f1e9aa71538, pool:0x7f1e443fcb60}, {unit_id:1021, tenant_id:1014, resource_pool_id:1021, unit_config:{unit_config_id:1021, name:"config_grafana_test_zone2_s_1_2_yyc", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f6f68, unit_config:0x7f1e9aa710d8, pool:0x7f1e443fc038}, {unit_id:1023, tenant_id:1016, resource_pool_id:1023, unit_config:{unit_config_id:1022, name:"config_temporalio_zone2_s_2_4_lsv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6db8, unit_config:0x7f1e9aa70ff8, pool:0x7f1e74dff688}, {unit_id:1027, tenant_id:1018, resource_pool_id:1027, unit_config:{unit_config_id:1025, name:"config_media_zone2_s_2_4_yzd", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6a58, unit_config:0x7f1e9aa70d58, pool:0x7f1e74dfe7a8}], not_grant_units=[], available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:34.322212] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=31] start to do tenant group unit balance(zone=zone3) [2025-08-15 10:15:34.322222] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=7] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.203.4.39:2882", disk_in_use_:3051356160, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:34.322233] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=7] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:34.322249] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=8] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:34.322261] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=7] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:34.322270] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone3) [2025-08-15 10:15:34.322275] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=4] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:34.322288] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84179688GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.39:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84179688GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:34.322319] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=24] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.203.4.39:2882", disk_in_use_:3051356160, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:34.322330] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=9] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.39:2882", disk_in_use_:3051356160, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:34.322340] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:34.322351] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=8] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1005, unit_config:{unit_config_id:1006, name:"config_opsone_zone3_s_2_4_vxv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7a60, unit_config:0x7f1e9aa71b58, pool:0x7f1c3beb32d0}, {unit_id:1007, tenant_id:1006, resource_pool_id:1007, unit_config:{unit_config_id:1007, name:"config_test_zone3_s_2_4_mui", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f78b0, unit_config:0x7f1e9aa71a78, pool:0x7f1c3beb2b60}, {unit_id:1012, tenant_id:1008, resource_pool_id:1012, unit_config:{unit_config_id:1012, name:"config_mcp_zone3_s_4_8_seu", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7478, unit_config:0x7f1e9aa71618, pool:0x7f1e443fd688}, {unit_id:1017, tenant_id:1012, resource_pool_id:1017, unit_config:{unit_config_id:1018, name:"config_oms_zone3_s_4_8_nvb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f72c8, unit_config:0x7f1e9aa71378, pool:0x7f1e443fcf18}, {unit_id:1020, tenant_id:1014, resource_pool_id:1020, unit_config:{unit_config_id:1020, name:"config_grafana_test_zone3_s_1_2_xbz", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7040, unit_config:0x7f1e9aa711b8, pool:0x7f1e443fc3f0}, {unit_id:1022, tenant_id:1016, resource_pool_id:1022, unit_config:{unit_config_id:1024, name:"config_temporalio_zone3_s_2_4_rxz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6e90, unit_config:0x7f1e9aa70e38, pool:0x7f1e74dffa40}, {unit_id:1025, tenant_id:1018, resource_pool_id:1025, unit_config:{unit_config_id:1026, name:"config_media_zone3_s_2_4_ukz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6c08, unit_config:0x7f1e9aa70c78, pool:0x7f1e74dfef18}], not_grant_units=[], available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:34.352883] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7849-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224133818979004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224133818979004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224133818979004, v:0}}], last_dump_ts_=1755224129321397, force_dump=false) [2025-08-15 10:15:34.352908] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7849-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.353098] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC042-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.353341] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DB-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.353678] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BA-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.353748] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C10-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224134246773000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224134246773000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224134246773000, v:0}}], last_dump_ts_=1755224129322609, force_dump=false) [2025-08-15 10:15:34.353771] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C10-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.356103] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7849-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.356169] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC042-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.356251] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DB-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.356577] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BA-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.356705] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C10-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224133818979004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224134319328001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224134319328001, v:0}}], last_dump_ts_=1755224129325860, force_dump=false) [2025-08-15 10:15:34.356723] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C10-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.358225] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC042-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.358306] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC042-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.358437] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DB-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.358438] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7849-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.358528] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7849-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.358531] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BA-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.358549] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DB-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.358644] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BA-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.358680] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C10-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.358774] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C10-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.414009] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7639-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224134216335000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224134319328001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224134319328001, v:0}}], last_dump_ts_=1755224129383417, force_dump=false) [2025-08-15 10:15:34.414033] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7639-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.416920] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7639-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224134216335000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224134319328001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224134319328001, v:0}}], last_dump_ts_=1755224129386578, force_dump=false) [2025-08-15 10:15:34.416948] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7639-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.418800] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7639-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.418875] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7639-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.440479] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6867-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.455711] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8843-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.455710] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A22-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.458248] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A22-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:34.458283] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8843-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:34.458266] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A22-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.458300] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8843-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.460012] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:34.460024] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:34.460031] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:34.461323] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:34.500207] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E71D-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.500271] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0AE-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.505887] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF212-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.523160] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D580A-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84179688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84570312GB}) [2025-08-15 10:15:34.523201] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D580A-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2025-08-15 10:15:34.532087] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:34.532101] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:34.532107] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:34.532695] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:34.532713] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:34.532723] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:34.533415] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:34.537979] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224132817610004, replayable_scn:1755224132817610004, readable_scn:1755224132817610004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224134535791, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224132817610004, replayable_scn:1755224132817610004, readable_scn:1755224132817610004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224134535791, ora_rowscn_=1755224134355055000, content_changed=true) [2025-08-15 10:15:34.538146] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:34.538157] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:34.538164] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:34.544257] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:34.544268] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:34.544274] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:34.549445] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:34.549456] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:34.549461] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:34.550763] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:34.559311] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:104) [17458][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=13] start checking lost replicas [2025-08-15 10:15:34.621834] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:34.623050] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224134621760, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:15:34.623073] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224139623072, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:15:34.681745] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38BA-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:34.684239] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38BA-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:34.689225] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C686A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.700548] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD7C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.706210] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:34.706222] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:34.706228] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:34.707500] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:34.732386] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959C-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.734914] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959C-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:34.734929] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959C-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.749667] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF055-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.749775] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD8B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.751773] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8868-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.754293] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8868-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:34.754310] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8868-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.758801] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C55-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1011, thread_idx=0) [2025-08-15 10:15:34.760040] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1011, only_for_display=false) [2025-08-15 10:15:34.761227] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:34.772630] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836A-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224134319328004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224134319328005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224134319328005, v:0}}], last_dump_ts_=1755224129742027, force_dump=false) [2025-08-15 10:15:34.772656] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836A-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.774886] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.775738] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836A-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.776054] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.777203] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.777226] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1011, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:34.777243] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.777254] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1011, ls_id:{id:1}, replicas:[{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:34.777372] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=93] build ls locality stat map(tenant_id=1011, ls_id={id:1}) [2025-08-15 10:15:34.777381] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.777393] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.777403] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.777414] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:34.777499] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=62] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.777514] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.777523] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.777529] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.777537] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3811, tenant_id=1011, ls_id={id:1}) [2025-08-15 10:15:34.777547] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:34.777835] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.777946] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:34.778627] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18588, tenant_id=1011, acc_dr_task=0) [2025-08-15 10:15:34.779046] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2F-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.779826] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1012, only_for_display=false) [2025-08-15 10:15:34.781014] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:34.781571] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2F-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:34.781588] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB2F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:34.794635] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.795789] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.796954] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.796978] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:34.796995] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.797002] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=5] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1}, replicas:[{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:34.797104] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=81] build ls locality stat map(tenant_id=1012, ls_id={id:1}) [2025-08-15 10:15:34.797113] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.797126] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.797137] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.797149] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:34.797226] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.797235] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=3] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.797245] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.797250] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.797259] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3785, tenant_id=1012, ls_id={id:1}) [2025-08-15 10:15:34.797269] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:34.798338] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:34.804577] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DC-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:34.805671] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5251-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:34.807064] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DC-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:34.808134] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5251-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:34.818983] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.820144] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.821406] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.821430] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:34.821447] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.821457] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1001}, replicas:[{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:34.821565] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=84] build ls locality stat map(tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:15:34.821576] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.821589] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.821598] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.821607] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:34.821685] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.821699] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.821707] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.821714] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.821721] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3874, tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:15:34.821730] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:34.823372] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:34.845916] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.847085] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.847462] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D0B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.848231] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.848254] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:34.848270] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.848280] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1002}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:34.848403] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=96] build ls locality stat map(tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:15:34.848415] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.848427] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.848436] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.848446] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:34.848523] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.848536] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.848543] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.848551] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.848559] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3790, tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:15:34.848567] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:34.850081] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:34.872362] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.873508] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.874656] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:34.874677] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:34.874691] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.874701] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1003}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:34.874809] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=84] build ls locality stat map(tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:15:34.874817] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.874829] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.874841] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:34.874850] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:34.874929] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.874948] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.874956] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.874964] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:34.874971] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3767, tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:15:34.874979] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:34.876634] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=97992, tenant_id=1012, acc_dr_task=0) [2025-08-15 10:15:34.878411] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8687-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:34.880932] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8687-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:34.881298] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F0-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:34.883776] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F0-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:34.897801] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B365-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:34.968478] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F96-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:34.970957] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F96-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:34.990821] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58D6-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.991527] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9B9-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:34.998450] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17740][T1_TimerWK3_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=5] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:35.008045] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB629A-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:35.010540] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB629A-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:35.026423] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [2619487][T1012_TimerWK2_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=7] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:35.026461] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [2619487][T1012_TimerWK2_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=26] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:35.026477] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [2619487][T1012_TimerWK2_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:35.027736] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [2619486][T1012_TimerWK1_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=30] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:35.100285] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED3-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1007, thread_idx=0) [2025-08-15 10:15:35.101579] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1007, only_for_display=false) [2025-08-15 10:15:35.102842] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:35.103224] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58D0-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.116642] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.117819] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.118973] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.118996] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1007, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:35.119011] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.119020] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1007, ls_id:{id:1}, replicas:[{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:35.119121] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=79] build ls locality stat map(tenant_id=1007, ls_id={id:1}) [2025-08-15 10:15:35.119131] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.119142] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.119151] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.119160] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:35.119237] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.119249] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.119256] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.119264] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.119270] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3790, tenant_id=1007, ls_id={id:1}) [2025-08-15 10:15:35.119278] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:35.120331] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18752, tenant_id=1007, acc_dr_task=0) [2025-08-15 10:15:35.121552] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1008, only_for_display=false) [2025-08-15 10:15:35.122747] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:35.135543] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0B1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.136478] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.137632] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.138789] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.138811] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:35.138825] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.138835] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1}, replicas:[{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:35.138934] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=78] build ls locality stat map(tenant_id=1008, ls_id={id:1}) [2025-08-15 10:15:35.138949] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.138960] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.138969] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.138979] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:35.139053] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=54] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.139062] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=3] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.139070] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.139078] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.139087] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3773, tenant_id=1008, ls_id={id:1}) [2025-08-15 10:15:35.139098] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:35.140155] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:35.147967] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF052-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.162340] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.163500] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.164664] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.164685] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:35.164699] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.164709] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1001}, replicas:[{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:35.164809] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=79] build ls locality stat map(tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:15:35.164817] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.164828] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.164834] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.164843] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:35.164926] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.164947] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.164955] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.164962] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.164969] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3785, tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:15:35.164983] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=10] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:35.166075] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:35.184703] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D09-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.186913] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.188060] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.189200] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.189223] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:35.189241] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.189252] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1002}, replicas:[{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:35.189359] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=83] build ls locality stat map(tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:15:35.189367] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.189381] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.189387] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.189399] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:35.189479] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=59] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.189493] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.189504] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.189510] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.189519] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3757, tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:15:35.189530] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:35.190625] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:35.199337] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9BE-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.211477] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.212651] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.213796] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:35.213819] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:35.213835] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.213847] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1003}, replicas:[{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:35.213961] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=89] build ls locality stat map(tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:15:35.213970] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.213983] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.213990] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:35.213997] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:35.214075] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.214090] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.214101] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.214106] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:35.214115] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3827, tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:15:35.214126] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:35.215197] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94853, tenant_id=1008, acc_dr_task=0) [2025-08-15 10:15:35.267507] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A09-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.268925] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF61-0-0] [lt=46] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.270087] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A09-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:35.270103] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A09-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.271470] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF61-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:35.271486] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF61-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.359368] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC043-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.359504] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DC-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.359516] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.359565] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BB-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.359665] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C11-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.362454] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.362479] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC043-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.362686] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C11-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.362696] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BB-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.362733] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DC-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.364472] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.364689] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.364781] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC043-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.364786] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C11-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.364838] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DC-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.364885] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C11-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.364897] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC043-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.364950] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DC-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.364966] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BB-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.365069] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BB-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.419870] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763A-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.422973] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763A-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.424937] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.425101] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763A-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.440557] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6868-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.459877] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224133317438004, replayable_scn:1755224133317438004, readable_scn:1755224133317438004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224135457240, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224133317438004, replayable_scn:1755224133317438004, readable_scn:1755224133317438004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224135457240, ora_rowscn_=1755224134774682000, content_changed=true) [2025-08-15 10:15:35.460067] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:35.460080] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:35.460086] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:35.461057] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8844-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.461073] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A23-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.463262] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E3D0-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}) [2025-08-15 10:15:35.463298] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E3D0-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=40, ddl_arg=NULL) [2025-08-15 10:15:35.463600] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8844-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:35.463635] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A23-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:35.463617] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8844-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.463652] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A23-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.500283] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E71E-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.500347] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0AF-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.531872] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224134319328001, replayable_scn:1755224134319328001, readable_scn:1755224134319328001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224135528805, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224134319328001, replayable_scn:1755224134319328001, readable_scn:1755224134319328001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224135528805, ora_rowscn_=1755224135421495000, content_changed=true) [2025-08-15 10:15:35.532055] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:35.532066] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:35.532074] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:35.532575] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224133818979005, replayable_scn:1755224133818979005, readable_scn:1755224133818979005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224135529858, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224133818979005, replayable_scn:1755224133818979005, readable_scn:1755224133818979005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224135529858, ora_rowscn_=1755224135361553000, content_changed=true) [2025-08-15 10:15:35.532742] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:35.532752] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:35.532760] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:35.538195] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:35.538210] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:35.538216] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:35.544152] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224133997766004, replayable_scn:1755224133997766004, readable_scn:1755224133997766004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224135541445, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224133997766004, replayable_scn:1755224133997766004, readable_scn:1755224133997766004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224135541445, ora_rowscn_=1755224135361553000, content_changed=true) [2025-08-15 10:15:35.544312] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:35.544324] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:35.544332] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:35.544341] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1008, cost_time_us=167, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:35.549353] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224134319328001, replayable_scn:1755224134319328001, readable_scn:1755224134319328001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224135546679, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224134319328001, replayable_scn:1755224134319328001, readable_scn:1755224134319328001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224135546679, ora_rowscn_=1755224135361553000, content_changed=true) [2025-08-15 10:15:35.549510] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:35.549523] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:35.549534] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:35.616732] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:35.618028] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224135616684, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:15:35.618059] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224140618057, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:15:35.679030] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [3413901][T1006_TimerWK4_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=19] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:35.689302] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C686B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.689634] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38BB-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:35.692122] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38BB-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:35.700624] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD7D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.704346] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC639-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.706077] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224133818979004, replayable_scn:1755224133818979004, readable_scn:1755224133818979004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224135703354, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224133818979004, replayable_scn:1755224133818979004, readable_scn:1755224133818979004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224135703354, ora_rowscn_=1755224135361553000, content_changed=true) [2025-08-15 10:15:35.706266] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:35.706278] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:35.706283] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:35.737649] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959D-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.740215] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959D-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:35.740234] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.749740] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF056-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.749846] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD8C-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.757022] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8869-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.759643] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8869-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:35.759662] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8869-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.778958] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836B-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.782089] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836B-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.784000] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.784123] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:35.784327] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB30-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.786911] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB30-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:35.786929] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB30-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:35.812489] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DD-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:35.813409] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5252-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:35.814973] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DD-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:35.815880] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5252-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:35.847539] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D0C-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.886371] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8688-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:35.888916] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8688-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:35.889088] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F1-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:35.891569] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F1-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:35.899231] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B366-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:35.941412] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:35.942659] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224135941339, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:15:35.942681] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224140942679, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:15:35.975623] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:35.976287] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F97-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:35.976846] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224135966186, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:15:35.976869] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224140976867, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:15:35.976888] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224140976886, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:35.978866] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F97-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:35.990895] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58D7-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:35.991603] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9BA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.015806] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB629B-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:36.018381] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB629B-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:36.103299] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58D1-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.135618] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0B2-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.148042] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF053-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.184271] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:36.184774] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D0A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.185504] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224136184207, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:15:36.185528] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224141185526, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:15:36.186931] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:36.188130] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224136186858, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:15:36.188149] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224141188148, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:36.199410] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9BF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.272857] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0A-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.274163] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF62-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.274357] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D45-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}) [2025-08-15 10:15:36.274397] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D45-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:15:36.275416] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0A-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:36.275435] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0A-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.276720] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF62-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:36.276736] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF62-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.283946] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=9] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:36.283980] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=23] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:36.283994] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=12] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:36.286487] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=9] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:36.286865] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:36.286882] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:36.292779] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:36.293965] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17347][TimerWK0_IngressService][T0][Y0-0000000000000000-0-0] [lt=1][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:36.365715] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.365838] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C12-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.365856] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC044-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.365920] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DD-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.366004] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BC-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224135821162005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224135821162005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224135821162005, v:0}}], last_dump_ts_=1755224131334822, force_dump=false) [2025-08-15 10:15:36.366031] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BC-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.368948] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784B-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.369007] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C12-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.369121] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BC-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224135821162005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224135821162005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224135821162005, v:0}}], last_dump_ts_=1755224131337872, force_dump=false) [2025-08-15 10:15:36.369147] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BC-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.369157] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC044-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224135821162004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224136322459001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224136322459001, v:0}}], last_dump_ts_=1755224131337340, force_dump=false) [2025-08-15 10:15:36.369179] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC044-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.369214] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DD-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.371013] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784B-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.371111] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C12-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.371123] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784B-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.371134] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BC-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.371209] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C12-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.371220] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BC-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.371258] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC044-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.371292] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DD-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.371353] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC044-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.371392] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DD-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.426098] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.429141] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763B-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.431080] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.431177] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.440627] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6869-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.460131] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:36.460143] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:36.460148] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:36.466431] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A24-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.466430] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8845-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.468982] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A24-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:36.469014] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8845-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:36.469001] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A24-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.469036] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8845-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.500357] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E71F-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.500422] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0B0-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.506205] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF213-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.523634] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D580B-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84570312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84960938GB}) [2025-08-15 10:15:36.523678] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D580B-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:15:36.532213] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:36.532226] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:36.532231] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:36.532822] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:36.532832] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:36.532837] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:36.538202] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:36.538216] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:36.538222] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:36.544405] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:36.544421] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:36.544433] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:36.549599] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:36.549612] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:36.549618] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:36.689378] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C686C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.697483] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38BC-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:36.699968] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38BC-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:36.700701] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD7E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.706328] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:36.706340] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:36.706346] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:36.742981] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959E-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.745541] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959E-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:36.745557] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.749817] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF057-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.749917] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD8D-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.762310] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886A-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.764810] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886A-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:36.764825] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886A-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.775586] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709536][T1014_TimerWK3_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=12] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:36.776555] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=13] mview_mds_op(tx_count=0) [2025-08-15 10:15:36.776569] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=10] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=867) [2025-08-15 10:15:36.781707] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475910][T1008_TimerWK2_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:36.785158] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.788236] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836C-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.789738] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB31-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.790270] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.790353] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:36.792365] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB31-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:36.792384] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB31-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:36.807225] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709637][T1013_DRService][T1013][YB420ACB0122-000633A341D55697-0-0] [lt=15] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1013, thread_idx=1) [2025-08-15 10:15:36.810975] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [709637][T1013_DRService][T1013][YB420ACB0122-000633A341D55698-0-0] [lt=14] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2435, tenant_id=1013, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:36.813297] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [709637][T1013_DRService][T1013][YB420ACB0122-000633A341D55698-0-0] [lt=13] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2303, tenant_id=1014, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:36.820281] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DE-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:36.821153] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5253-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:36.822798] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DE-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:36.823663] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5253-0-0] [lt=9] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:36.825770] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:36.826984] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224136825742, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:15:36.827006] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224141827004, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:15:36.847607] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D0D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.894257] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8689-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:36.896844] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F2-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:36.897974] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8689-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:36.899335] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F2-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:36.900672] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B367-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:36.911282] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=11] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224136535934, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224135248093000, replayable_scn:1755224135248093000, readable_scn:1755224135248093000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:36.984282] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F98-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:36.986770] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F98-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:36.990967] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58D8-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.991679] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9BB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:36.993772] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [709633][T1013_BackupDat][T1013][YB420ACB0122-000633A33E8598BD-0-0] [lt=12] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:37.023742] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB629C-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:37.026288] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB629C-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:37.103339] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58D2-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.121408] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=16] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:37.135692] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0B3-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.148117] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF054-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.184849] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D0B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.199466] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9C0-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.201306] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:37.255689] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:37.278150] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0B-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.279446] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF63-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.280719] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0B-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:37.280736] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0B-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.282005] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF63-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:37.282022] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF63-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.315630] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487606][T1004_TimerWK3_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=6] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:37.329805] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B589E-0-0] [lt=11] [RESTORE] restore scheduler quit [2025-08-15 10:15:37.330540] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=6] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:37.331088] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B589E-0-0] [lt=14] get targets(ret=0, jobs=[], exec_tenant_id=1007, sql=select * from __all_recover_table_job) [2025-08-15 10:15:37.331793] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224137330464, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:15:37.331818] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224142331817, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:15:37.332286] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B589F-0-0] [lt=14] get targets(ret=0, jobs=[], exec_tenant_id=1007, sql=select * from __all_import_table_job) [2025-08-15 10:15:37.332310] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B58A0-0-0] [lt=14][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:37.332390] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B58A1-0-0] [lt=10] [RESTORE] restore scheduler start [2025-08-15 10:15:37.332398] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B58A2-0-0] [lt=6] [RESTORE] try process restore job [2025-08-15 10:15:37.333548] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [476029][T1007_REST_SER][T1007][YB420ACB0122-00063382125B58A2-0-0] [lt=5] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:15:37.372177] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.372177] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BD-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.372239] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C13-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.372260] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC045-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224136822999005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224137323158001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224137323158001, v:0}}], last_dump_ts_=1755224132340424, force_dump=false) [2025-08-15 10:15:37.372290] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC045-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.372312] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DE-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.375515] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C13-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.375521] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DE-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.375643] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC045-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.375745] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784C-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224136822999005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224136822999005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224136822999005, v:0}}], last_dump_ts_=1755224132343692, force_dump=false) [2025-08-15 10:15:37.375770] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.376084] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BD-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.377606] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C13-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.377649] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC045-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.377674] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DE-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.377708] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C13-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.377764] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC045-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.377789] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DE-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.377814] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.377906] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.378043] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BD-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.378124] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BD-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.416843] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:37.418114] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224137416767, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:15:37.418136] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224142418134, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:15:37.421555] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E5F-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224137421072000, start_id=1755224137421072000, end_id=1755224137421072001) [2025-08-15 10:15:37.422232] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E5F-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224137421072000, v:0}, tenant_id=1017) [2025-08-15 10:15:37.432190] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.435206] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763C-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.437134] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.437233] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763C-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.440699] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C686A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.460196] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:37.460207] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:37.460212] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:37.463679] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A072E-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}) [2025-08-15 10:15:37.463721] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A072E-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:37.471761] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8846-0-0] [lt=40] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.471759] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A25-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.474285] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A25-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:37.474296] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8846-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:37.474302] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A25-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.474309] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8846-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.500431] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E720-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.500495] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0B1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.532191] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:37.532205] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:37.532210] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:37.532867] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:37.532879] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:37.532884] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:37.534186] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:37.538156] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224136245772000, replayable_scn:1755224136245772000, readable_scn:1755224136245772000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224137535829, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224136245772000, replayable_scn:1755224136245772000, readable_scn:1755224136245772000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224137535829, ora_rowscn_=1755224137373699000, content_changed=true) [2025-08-15 10:15:37.538342] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:37.538354] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:37.538360] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:37.544410] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:37.544420] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:37.544425] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:37.545718] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:37.549609] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:37.549621] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:37.549627] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:37.556211] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E68-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224137555842000, start_id=1755224137555842000, end_id=1755224137555842001) [2025-08-15 10:15:37.556867] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E68-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224137555842000, v:0}, tenant_id=1015) [2025-08-15 10:15:37.557187] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66A-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224137556896000, start_id=1755224137556896000, end_id=1755224137556896001) [2025-08-15 10:15:37.557911] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66A-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224137556896000, v:0}, tenant_id=1005) [2025-08-15 10:15:37.689452] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C686D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.700778] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD7F-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.705337] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38BD-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:37.706379] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:37.706394] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:37.706400] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:37.707834] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38BD-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:37.709270] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC63A-0-0] [lt=13] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.748314] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959F-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.749889] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF058-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.749991] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD8E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.750883] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959F-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:37.750900] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D959F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.767524] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886B-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.770036] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886B-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:37.770052] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886B-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.785697] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36C8-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224137785020000, start_id=1755224137785020000, end_id=1755224137785020001) [2025-08-15 10:15:37.786349] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36C8-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224137785020000, v:0}, tenant_id=1011) [2025-08-15 10:15:37.791363] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836D-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.794520] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.795129] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB32-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.795211] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:37.796459] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224137795136, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:15:37.796483] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224142796481, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:15:37.796595] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.796685] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:37.796693] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:37.797653] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB32-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:37.797669] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB32-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:37.797870] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224137796621, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:15:37.797892] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224142797891, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:15:37.828085] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DF-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:37.828929] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5254-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:37.829329] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:351) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DF-0-0] [lt=15] log restore source is empty, just skip(ret=-4018, user_tenant_id=1012) [2025-08-15 10:15:37.831413] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5254-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:37.832236] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8DF-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:37.847679] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D0E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.902159] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B368-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:37.903395] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB868A-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:37.903644] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB0-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224137902770000, start_id=1755224137902770000, end_id=1755224137902770001) [2025-08-15 10:15:37.904297] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB0-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224137902770000, v:0}, tenant_id=1013) [2025-08-15 10:15:37.904661] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F3-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:37.905889] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB868A-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:37.907195] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F3-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:37.958612] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:37.959828] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224137958537, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:15:37.959851] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224142959850, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:15:37.991043] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58D9-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.991751] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9BC-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:37.992099] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F99-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:37.994609] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F99-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:38.031660] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB629D-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:38.034152] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB629D-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:38.044439] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:38.045639] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224138044365, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:15:38.045664] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224143045662, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:15:38.081698] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B0-0-0] [lt=17] get number(ret=0, service_type_=0, range=1, base_id=1755224138081335000, start_id=1755224138081335000, end_id=1755224138081335001) [2025-08-15 10:15:38.082527] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B0-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224138081335000, v:0}, tenant_id=1012) [2025-08-15 10:15:38.101732] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463B8-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224138101352000, start_id=1755224138101352000, end_id=1755224138101352001) [2025-08-15 10:15:38.102582] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463B8-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224138101352000, v:0}, tenant_id=1) [2025-08-15 10:15:38.103419] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58D3-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.130451] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF826F-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1005, thread_idx=0) [2025-08-15 10:15:38.131708] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1005, only_for_display=false) [2025-08-15 10:15:38.132962] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:38.135763] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0B4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.146806] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.147977] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.148147] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF055-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.149107] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.149131] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1005, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:38.149144] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.149156] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1005, ls_id:{id:1}, replicas:[{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:38.149260] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=82] build ls locality stat map(tenant_id=1005, ls_id={id:1}) [2025-08-15 10:15:38.149269] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.149282] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.149293] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.149305] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:38.149382] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.149399] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.149413] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=11] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.149419] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.149429] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3769, tenant_id=1005, ls_id={id:1}) [2025-08-15 10:15:38.149441] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:38.150495] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18787, tenant_id=1005, acc_dr_task=0) [2025-08-15 10:15:38.151698] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1006, only_for_display=false) [2025-08-15 10:15:38.152902] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:38.167300] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.168446] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.169585] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.169610] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:38.169627] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.169639] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1}, replicas:[{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:38.169743] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=82] build ls locality stat map(tenant_id=1006, ls_id={id:1}) [2025-08-15 10:15:38.169751] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.169766] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.169777] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.169787] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:38.169864] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.169879] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.169888] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.169895] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.169904] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3787, tenant_id=1006, ls_id={id:1}) [2025-08-15 10:15:38.169915] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:38.171002] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:38.184919] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D0C-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.191693] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.192848] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.194002] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.194025] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:38.194038] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.194049] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1001}, replicas:[{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:38.194153] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=81] build ls locality stat map(tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:15:38.194161] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.194174] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.194185] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.194196] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:38.194273] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.194287] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.194297] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.194306] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.194312] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3739, tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:15:38.194322] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:38.195376] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:38.199538] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9C1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.215953] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.217098] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.218228] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.218252] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:38.218270] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.218281] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1002}, replicas:[{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:38.218403] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=95] build ls locality stat map(tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:15:38.218415] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.218429] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.218438] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.218449] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:38.218528] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.218543] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.218553] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.218561] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.218572] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3749, tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:15:38.218582] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:38.219656] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:38.240237] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.241373] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.242532] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.242554] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:38.242569] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.242578] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1003}, replicas:[{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:38.242678] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=78] build ls locality stat map(tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:15:38.242685] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.242696] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.242704] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.242710] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:38.242785] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.242797] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.242807] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.242817] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.242823] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3719, tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:15:38.242832] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:38.243894] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93384, tenant_id=1006, acc_dr_task=0) [2025-08-15 10:15:38.269302] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [476159][T1008_MergeSche][T1008][YB420ACB0122-000633821382DA59-0-0] [lt=69] zone_merge_mgr is already loaded(tenant_id=1008, global_merge_info={tenant_id:1008, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402107469, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194714424606, need_update:false}}, zone_merge_infos=[{tenant_id:1008, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194714425145, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402139417, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1008, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194714426903, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402144931, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1008, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194714428639, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402152998, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402023059000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:15:38.274860] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D47-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}) [2025-08-15 10:15:38.274906] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D47-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:15:38.283425] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0C-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.284706] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF64-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.285946] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0C-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:38.285962] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0C-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.287213] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF64-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:38.287228] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF64-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.378719] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C14-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.378731] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DF-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224137824985004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224138325413001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224138325413001, v:0}}], last_dump_ts_=1755224133346789, force_dump=false) [2025-08-15 10:15:38.378757] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DF-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.378762] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC046-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.378890] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784D-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.379094] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BE-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.381843] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C14-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.381862] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC046-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.381896] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.381934] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DF-0-0] [lt=9] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224137824985004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224138325413001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224138325413001, v:0}}], last_dump_ts_=1755224133350097, force_dump=false) [2025-08-15 10:15:38.381968] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DF-0-0] [lt=26] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.381972] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BE-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.383925] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C14-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.383989] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BE-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.384018] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DF-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.384035] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC046-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.384038] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.384052] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C14-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.384120] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BE-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.384130] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.384151] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC046-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.384169] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76DF-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.438168] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.440778] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C686B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.441189] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.443123] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.443286] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.460068] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224136322459005, replayable_scn:1755224136322459005, readable_scn:1755224136322459005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224138456905, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224136322459005, replayable_scn:1755224136322459005, readable_scn:1755224136322459005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224138456905, ora_rowscn_=1755224137793445000, content_changed=true) [2025-08-15 10:15:38.460253] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:38.460265] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:38.460270] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:38.477119] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A26-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.477119] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8847-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.479717] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8847-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:38.479717] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A26-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:38.479735] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A26-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.479736] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8847-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.500501] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E721-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.500563] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0B2-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.506478] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF214-0-0] [lt=16] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.524102] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D127E-0-0] [lt=14] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.84960938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.85351562GB}) [2025-08-15 10:15:38.524141] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D127E-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2025-08-15 10:15:38.532068] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224137001592004, replayable_scn:1755224137001592004, readable_scn:1755224137001592004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224138528522, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224137001592004, replayable_scn:1755224137001592004, readable_scn:1755224137001592004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224138528522, ora_rowscn_=1755224138440055000, content_changed=true) [2025-08-15 10:15:38.532243] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:38.532256] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:38.532261] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:38.532750] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=12] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224136822999005, replayable_scn:1755224136822999005, readable_scn:1755224136822999005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224138529576, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224136822999005, replayable_scn:1755224136822999005, readable_scn:1755224136822999005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224138529576, ora_rowscn_=1755224138381111000, content_changed=true) [2025-08-15 10:15:38.532907] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:38.532926] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:38.532934] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:38.538315] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:38.538326] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:38.538333] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:38.539619] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:38.544282] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224137001592004, replayable_scn:1755224137001592004, readable_scn:1755224137001592004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224138541163, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224137001592004, replayable_scn:1755224137001592004, readable_scn:1755224137001592004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224138541163, ora_rowscn_=1755224138380064000, content_changed=true) [2025-08-15 10:15:38.544453] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:38.544463] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:38.544470] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:38.544484] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1008, cost_time_us=180, return_code_array=[0, 0, 0], broadcast_times=2) [2025-08-15 10:15:38.549559] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224137323158001, replayable_scn:1755224137323158001, readable_scn:1755224137323158001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224138546420, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224137323158001, replayable_scn:1755224137323158001, readable_scn:1755224137323158001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224138546420, ora_rowscn_=1755224138380064000, content_changed=true) [2025-08-15 10:15:38.549720] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:38.549732] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:38.549737] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:38.592193] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEE-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1, thread_idx=0) [2025-08-15 10:15:38.593429] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-08-15 10:15:38.594637] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:38.602350] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:38.603619] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224138602273, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:15:38.603647] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224143603646, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:15:38.608372] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.609535] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.610684] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:38.610707] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:38.610724] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.610735] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:38.610839] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=81] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-08-15 10:15:38.610847] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.610860] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.610872] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=10] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:38.610884] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:38.610975] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=67] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.610991] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.611000] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.611010] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:38.611015] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3793, tenant_id=1, ls_id={id:1}) [2025-08-15 10:15:38.611025] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:38.612143] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18715, tenant_id=1, acc_dr_task=0) [2025-08-15 10:15:38.689533] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C686E-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.700849] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD80-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.706288] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224136822999005, replayable_scn:1755224136822999005, readable_scn:1755224136822999005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224138702918, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224136822999005, replayable_scn:1755224136822999005, readable_scn:1755224136822999005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224138702918, ora_rowscn_=1755224138380064000, content_changed=true) [2025-08-15 10:15:38.706478] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:38.706489] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:38.706497] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:38.713146] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38BE-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:38.715611] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38BE-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:38.738908] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E58-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224138738695000, start_id=1755224138738695000, end_id=1755224138738695001) [2025-08-15 10:15:38.739539] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E58-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224138738695000, v:0}, tenant_id=1016) [2025-08-15 10:15:38.749961] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF059-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.750064] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD8F-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.753581] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A0-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.756165] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A0-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:38.756182] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A0-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.772732] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886C-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.775323] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886C-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:38.775341] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886C-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.796449] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [438124][T1005_BackupDat][T1005][YB420ACB0122-0006338207177DA3-0-0] [lt=16] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:38.797722] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.800439] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB33-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.800843] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836E-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224138325413004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224138325413004, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224138325413004, v:0}}], last_dump_ts_=1755224133769579, force_dump=false) [2025-08-15 10:15:38.800864] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836E-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.802829] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.802922] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:38.803002] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB33-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:38.803020] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB33-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:38.836698] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5255-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:38.837451] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E0-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:38.839199] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5255-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:38.839951] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E0-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:38.847750] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D0F-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.903600] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B369-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:38.906387] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619613][T1011_DRService][T1011][YB420ACB0122-0006338CC425E900-0-0] [lt=15] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1011, thread_idx=1) [2025-08-15 10:15:38.909959] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [2619613][T1011_DRService][T1011][YB420ACB0122-0006338CC425E901-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2337, tenant_id=1011, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:38.911129] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB868B-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:38.912286] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [2619613][T1011_DRService][T1011][YB420ACB0122-0006338CC425E901-0-0] [lt=16] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2305, tenant_id=1012, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:38.912968] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F4-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:38.913606] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB868B-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:38.916175] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F4-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:38.991123] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58DA-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.991824] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9BD-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:38.999979] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F9A-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:39.002478] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F9A-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:39.039759] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB629E-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:39.042266] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB629E-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:39.100033] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E61-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1003, thread_idx=0) [2025-08-15 10:15:39.101294] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1003, only_for_display=false) [2025-08-15 10:15:39.102511] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:39.103500] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58D4-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.116254] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.117411] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.118557] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.118580] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1003, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:39.118596] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.118607] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1003, ls_id:{id:1}, replicas:[{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:39.118716] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=84] build ls locality stat map(tenant_id=1003, ls_id={id:1}) [2025-08-15 10:15:39.118729] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.118742] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.118753] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.118760] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:39.118842] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.118857] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.118867] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.118874] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.118883] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3790, tenant_id=1003, ls_id={id:1}) [2025-08-15 10:15:39.118893] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:39.119995] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18700, tenant_id=1003, acc_dr_task=0) [2025-08-15 10:15:39.121208] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1004, only_for_display=false) [2025-08-15 10:15:39.122409] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:39.135831] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0B5-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.136111] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.137276] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.138435] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.138460] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:39.138477] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.138489] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:39.138593] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=81] build ls locality stat map(tenant_id=1004, ls_id={id:1}) [2025-08-15 10:15:39.138601] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.138614] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.138621] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.138631] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:39.138707] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.138722] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.138732] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.138742] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.138748] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3800, tenant_id=1004, ls_id={id:1}) [2025-08-15 10:15:39.138758] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:39.139841] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:39.148218] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF056-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.152793] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64B-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224139152102000, start_id=1755224139152102000, end_id=1755224139152102001) [2025-08-15 10:15:39.153688] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64B-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224139152102000, v:0}, tenant_id=1006) [2025-08-15 10:15:39.160551] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.161707] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.162851] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.162872] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:39.162886] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.162896] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:39.163007] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=87] build ls locality stat map(tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:15:39.163014] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.163025] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.163031] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.163041] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:39.163117] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.163129] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.163137] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.163147] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.163155] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3758, tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:15:39.163163] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:39.164222] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:39.184976] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D0D-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.185138] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.186286] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.187441] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.187465] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:39.187484] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.187495] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1002}, replicas:[{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:39.187600] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=82] build ls locality stat map(tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:15:39.187609] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.187623] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.187630] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.187641] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:39.187718] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.187733] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.187743] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.187749] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.187758] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3773, tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:15:39.187769] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:39.188965] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:39.199620] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9C2-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.210373] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.211545] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.212688] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.212708] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:39.212723] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.212732] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1003}, replicas:[{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:39.212834] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=80] build ls locality stat map(tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:15:39.212842] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.212852] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.212861] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.212871] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:39.212952] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.212964] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.212972] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.212979] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.212986] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3761, tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:15:39.212994] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:39.214070] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94057, tenant_id=1004, acc_dr_task=0) [2025-08-15 10:15:39.217614] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A4-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224139217274000, start_id=1755224139217274000, end_id=1755224139217274001) [2025-08-15 10:15:39.218435] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A4-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224139217274000, v:0}, tenant_id=1008) [2025-08-15 10:15:39.288644] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0D-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.289867] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF65-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.291150] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0D-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:39.291165] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0D-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.292383] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF65-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:39.292398] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF65-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.298320] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:39.298340] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:39.304448] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:39.305632] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:39.385080] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784E-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224138826087004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224139326589001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224139326589001, v:0}}], last_dump_ts_=1755224134352882, force_dump=false) [2025-08-15 10:15:39.385106] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784E-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.385177] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E0-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.385188] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC047-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.385181] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C15-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224139249792000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224139249792000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224139249792000, v:0}}], last_dump_ts_=1755224134353747, force_dump=false) [2025-08-15 10:15:39.385204] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C15-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.385315] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BF-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.388278] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BF-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.388287] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.388329] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E0-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.388386] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC047-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.388526] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C15-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224139218328000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224139326589001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224139326589001, v:0}}], last_dump_ts_=1755224134356704, force_dump=false) [2025-08-15 10:15:39.388547] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C15-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.390334] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.390330] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E0-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.390339] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BF-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.390405] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC047-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.390417] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.390425] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413BF-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.390509] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC047-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.390509] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E0-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.391049] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C15-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.391133] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C15-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.419681] INFO [RS] gc_mview (ob_mview_maintenance_task.cpp:267) [1487604][T1004_TimerWK1_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=11] mview maintenance task gc mview success(tenant_id_=1004, round_=116350, gc_mview_cost_us_=1224, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:15:39.430176] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BA-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1017, thread_idx=0) [2025-08-15 10:15:39.431496] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1017, only_for_display=false) [2025-08-15 10:15:39.432789] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:39.440856] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C686C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.444317] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763E-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224139326589001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224139326589001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224139326589001, v:0}}], last_dump_ts_=1755224134414008, force_dump=false) [2025-08-15 10:15:39.444345] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763E-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.447013] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.447450] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763E-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224139398013002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224139398013002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224139398013002, v:0}}], last_dump_ts_=1755224134416920, force_dump=false) [2025-08-15 10:15:39.447475] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763E-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.448213] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.449395] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763E-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.449423] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.449444] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1017, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:39.449460] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.449515] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.449470] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1017, ls_id:{id:1}, replicas:[{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:39.449571] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=78] build ls locality stat map(tenant_id=1017, ls_id={id:1}) [2025-08-15 10:15:39.449579] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.449591] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.449599] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.449607] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:39.449684] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.449696] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.449703] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.449710] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.449717] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3922, tenant_id=1017, ls_id={id:1}) [2025-08-15 10:15:39.449725] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:39.450790] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19294, tenant_id=1017, acc_dr_task=0) [2025-08-15 10:15:39.452063] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=8] start try tenant disaster recovery(tenant_id=1018, only_for_display=false) [2025-08-15 10:15:39.453309] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:39.460305] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:39.460316] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:39.460321] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:39.461608] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:39.464120] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A072F-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}) [2025-08-15 10:15:39.464162] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A072F-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:39.467556] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.468749] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.469964] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.469993] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:39.470015] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=14] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.470029] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1}, replicas:[{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:39.470146] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=91] build ls locality stat map(tenant_id=1018, ls_id={id:1}) [2025-08-15 10:15:39.470159] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.470173] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.470184] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.470195] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:39.470272] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.470287] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.470297] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.470303] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.470312] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3956, tenant_id=1018, ls_id={id:1}) [2025-08-15 10:15:39.470323] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:39.471441] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:39.482466] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8848-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.482596] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A27-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.485050] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8848-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:39.485068] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8848-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.485135] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A27-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:39.485153] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A27-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.492855] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.494019] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.495178] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.495199] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:39.495214] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.495223] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1001}, replicas:[{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:39.495324] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=78] build ls locality stat map(tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:15:39.495331] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.495340] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=6] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.495348] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.495357] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:39.495432] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.495443] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.495450] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.495457] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.495463] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3763, tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:15:39.495472] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:39.496580] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:39.500580] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E722-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.500601] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0B3-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.517866] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.519086] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.520293] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.520317] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:39.520335] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.520346] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1002}, replicas:[{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:39.520448] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=79] build ls locality stat map(tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:15:39.520456] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.520470] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.520480] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.520488] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:39.520571] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.520585] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.520594] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.520599] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.520608] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3905, tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:15:39.520615] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:39.521696] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:39.532319] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:39.532329] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:39.532334] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:39.532894] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:39.532909] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:39.532916] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:39.533626] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:39.538333] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:39.538348] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:39.538354] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:39.543081] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.544275] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.544505] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:39.544526] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:39.544540] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:39.545445] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:39.545468] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:39.545485] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.545497] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1003}, replicas:[{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:39.545601] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=81] build ls locality stat map(tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:15:39.545609] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.545622] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.545633] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:39.545645] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:39.545722] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.545736] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.545747] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.545753] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:39.545761] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3899, tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:15:39.545772] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:39.546900] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=96097, tenant_id=1018, acc_dr_task=0) [2025-08-15 10:15:39.549721] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:39.549733] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:39.549739] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:39.550998] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:39.623146] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:39.624363] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224139623072, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:15:39.624388] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224144624386, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:15:39.689611] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C686F-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.700929] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD81-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.706461] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:39.706480] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:39.706493] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:39.707860] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:39.713272] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E4-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224139712407000, start_id=1755224139712407000, end_id=1755224139712407001) [2025-08-15 10:15:39.714007] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E4-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224139712407000, v:0}, tenant_id=1003) [2025-08-15 10:15:39.714235] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC63B-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.720891] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38BF-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:39.723358] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38BF-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:39.750025] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF05A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.750139] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD90-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.758857] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A1-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.761440] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A1-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:39.761458] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A1-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.778036] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886D-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.780578] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886D-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:39.780597] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.803958] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836F-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224139326589005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224139326589005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224139326589005, v:0}}], last_dump_ts_=1755224134772629, force_dump=false) [2025-08-15 10:15:39.803993] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836F-0-0] [lt=25] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.806567] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB34-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.807949] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836F-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.810008] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB34-0-0] [lt=82] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:39.810030] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB34-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:39.810521] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.810600] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D836F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:39.844474] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5256-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:39.845038] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E1-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:39.846960] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5256-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:39.847526] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E1-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:39.847824] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D10-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.853533] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C1-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224139853407000, start_id=1755224139853407000, end_id=1755224139853407001) [2025-08-15 10:15:39.854325] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C1-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224139853407000, v:0}, tenant_id=1004) [2025-08-15 10:15:39.905018] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B36A-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:39.912878] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224139535770, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224138325413001, replayable_scn:1755224138325413001, readable_scn:1755224138325413001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:39.918808] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB868C-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:39.921289] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB868C-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:39.921454] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F5-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:39.923928] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F5-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:39.991197] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58DB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.991899] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9BE-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:39.999758] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17739][T1_TimerWK2_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:40.007923] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F9B-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:40.010432] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F9B-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:40.047752] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB629F-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:40.050283] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB629F-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:40.103577] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58D5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.135907] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0B6-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.148292] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF057-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.185048] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D0E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.185060] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C2-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224140184264000, start_id=1755224140184264000, end_id=1755224140184264001) [2025-08-15 10:15:40.185761] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C2-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224140184264000, v:0}, tenant_id=1007) [2025-08-15 10:15:40.199694] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9C3-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.204114] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727239][T1015_DRService][T1015][YB420ACB0122-000633A351E5562C-0-0] [lt=13] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1015, thread_idx=1) [2025-08-15 10:15:40.207751] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [727239][T1015_DRService][T1015][YB420ACB0122-000633A351E5562D-0-0] [lt=14] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2346, tenant_id=1015, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:40.210101] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [727239][T1015_DRService][T1015][YB420ACB0122-000633A351E5562D-0-0] [lt=16] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2329, tenant_id=1016, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:40.275365] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA08-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88867188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}) [2025-08-15 10:15:40.275406] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA08-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:40.293852] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0E-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.295073] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF66-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.296398] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0E-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:40.296416] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0E-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.297654] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF66-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:40.297671] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF66-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.322109] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [438021][T1006_TimerWK0_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=15] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:40.322143] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [438021][T1006_TimerWK0_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=22] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:40.322159] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [438021][T1006_TimerWK0_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:40.323398] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [438024][T1006_TimerWK3_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=13] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:40.338264] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [17712][T1_BackupDataSr][T1][YB420ACB0122-0006337F03F7B80F-0-0] [lt=15] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:40.391502] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C0-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.391512] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC048-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.391511] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.391525] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E1-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.392084] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C16-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.394571] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC048-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.394570] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C0-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.394592] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784F-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.394629] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E1-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.394999] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C16-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.396541] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC048-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.396542] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C0-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.396640] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C0-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.396649] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784F-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.396655] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC048-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.396654] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E1-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.396734] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E1-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.396819] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB784F-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.396935] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C16-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.397057] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C16-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.440909] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C686D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.450504] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.453376] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763F-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.455251] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763F-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.455349] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D763F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.460307] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:40.460318] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:40.460323] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:40.487746] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8849-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.487830] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A28-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.490299] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8849-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:40.490324] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A28-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:40.490315] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8849-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.490339] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A28-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.500659] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E723-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.500673] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0B4-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.506748] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF215-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.524578] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A76-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.85351562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.85742188GB}) [2025-08-15 10:15:40.524618] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A76-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2025-08-15 10:15:40.532275] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:40.532292] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:40.532297] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:40.532989] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:40.533000] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:40.533005] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:40.534103] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [728185][T1018_MergeSche][T1018][YB420ACB0122-000633A35692D609-0-0] [lt=66] zone_merge_mgr is already loaded(tenant_id=1018, global_merge_info={tenant_id:1018, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194404562795, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194716733156, need_update:false}}, zone_merge_infos=[{tenant_id:1018, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194716735248, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194404594348, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1018, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194716738764, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194404599542, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1018, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194716740619, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194404603540, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194404248531000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:15:40.538827] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224139326589001, replayable_scn:1755224139326589001, readable_scn:1755224139326589001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224140535340, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224139326589001, replayable_scn:1755224139326589001, readable_scn:1755224139326589001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224140535340, ora_rowscn_=1755224140393520000, content_changed=true) [2025-08-15 10:15:40.538988] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:40.538998] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:40.539006] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:40.544441] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:40.544451] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:40.544457] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:40.549718] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:40.549728] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:40.549733] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:40.595171] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17716][T1_DRService][T1][YB420ACB0122-0006337F0497B7A0-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1, thread_idx=1) [2025-08-15 10:15:40.598685] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [17716][T1_DRService][T1][YB420ACB0122-0006337F0497B7A1-0-0] [lt=12] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2290, tenant_id=1, need_clean_task=true, service_epoch_to_check=2) [2025-08-15 10:15:40.618133] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:40.619331] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224140618057, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:15:40.619353] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224145619351, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:15:40.684381] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438024][T1006_TimerWK3_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=8] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:15:40.689689] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6870-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.701018] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD82-0-0] [lt=25] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.706560] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:40.706571] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:40.706576] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:40.728636] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C0-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:40.732251] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C0-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:40.750099] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF05B-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.750210] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD91-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.764151] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A2-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.766728] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A2-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:40.766744] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A2-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.783370] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886E-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.785924] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886E-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:40.785949] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886E-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.811615] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8370-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.813449] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB35-0-0] [lt=43] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.815482] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8370-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.816878] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB35-0-0] [lt=85] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:40.816901] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB35-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:40.817870] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8370-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.817952] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8370-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:40.847897] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D11-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.852262] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5257-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:40.852751] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E2-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:40.854737] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5257-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:40.855179] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E2-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:40.906543] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B36B-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:40.926560] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB868D-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:40.929055] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB868D-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:40.929121] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F6-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:40.931573] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:40.942753] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:40.943983] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224140942679, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:15:40.944007] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224145944005, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:15:40.985479] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:40.986693] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224140976867, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:15:40.986717] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224145986716, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:15:40.986733] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224145986732, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:40.991269] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58DC-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:40.991974] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9BF-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.015816] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F9C-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:41.018373] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F9C-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:41.055720] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A0-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:41.058302] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A0-0-0] [lt=9] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:41.103649] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58D6-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.135981] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0B7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.148366] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF058-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.185125] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D0F-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.185600] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:41.186805] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224141185526, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:15:41.186827] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224146186825, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:15:41.188220] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:41.189403] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224141188148, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:15:41.189422] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224146189421, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:41.199775] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9C4-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.299118] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0F-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.300347] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF67-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.301647] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0F-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:41.301667] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A0F-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.302868] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF67-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:41.302886] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF67-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.381614] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D1-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1015, thread_idx=0) [2025-08-15 10:15:41.382888] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1015, only_for_display=false) [2025-08-15 10:15:41.384103] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:41.395946] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E4F-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224141395038000, start_id=1755224141395038000, end_id=1755224141395038001) [2025-08-15 10:15:41.396376] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1080) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E4F-0-0] [lt=15] [PALF STAT GROUP LOG INFO](palf_id=1, self="10.203.1.34:2882", role="LEADER", total_group_log_cnt=11, avg_log_batch_cnt=1, total_group_log_size=2880, avg_group_log_size=261) [2025-08-15 10:15:41.396395] INFO [PALF] submit_log (palf_handle_impl.cpp:446) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E4F-0-0] [lt=14] [PALF STAT APPEND DATA SIZE](this={palf_id:1, self:"10.203.1.34:2882", has_set_deleted:false}, append size=2611) [2025-08-15 10:15:41.396757] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E4F-0-0] [lt=9] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224141395038000, v:0}, tenant_id=1018) [2025-08-15 10:15:41.397646] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC049-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.397631] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C1-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224140828381005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224140828381005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224140828381005, v:0}}], last_dump_ts_=1755224136366003, force_dump=false) [2025-08-15 10:15:41.397666] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C1-0-0] [lt=24] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.397671] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E2-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.397760] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7850-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.397883] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.397926] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C17-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.399060] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.400258] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.400286] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1015, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:41.400304] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.400316] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1015, ls_id:{id:1}, replicas:[{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:41.400430] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=89] build ls locality stat map(tenant_id=1015, ls_id={id:1}) [2025-08-15 10:15:41.400439] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.400460] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=16] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.400469] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.400480] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:41.400560] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.400574] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.400583] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.400590] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.400595] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3878, tenant_id=1015, ls_id={id:1}) [2025-08-15 10:15:41.400593] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E2-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.400600] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:41.400755] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7850-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.400876] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC049-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224140828381005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224141328778001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224141328778001, v:0}}], last_dump_ts_=1755224136369157, force_dump=false) [2025-08-15 10:15:41.400899] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC049-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.400903] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C17-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.401416] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C1-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224140828381005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224140828381005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224140828381005, v:0}}], last_dump_ts_=1755224136369120, force_dump=false) [2025-08-15 10:15:41.401438] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C1-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.401778] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18889, tenant_id=1015, acc_dr_task=0) [2025-08-15 10:15:41.402610] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E2-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.402719] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E2-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.402879] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7850-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.402902] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC049-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.402936] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C17-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.402988] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7850-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.403023] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC049-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.403054] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1016, only_for_display=false) [2025-08-15 10:15:41.403145] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C17-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.403366] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C1-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.403453] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C1-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.404312] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:41.418010] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.419148] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.420290] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.420310] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:41.420329] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.420338] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1}, replicas:[{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:41.420437] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=77] build ls locality stat map(tenant_id=1016, ls_id={id:1}) [2025-08-15 10:15:41.420447] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.420457] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.420465] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.420475] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:41.420550] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.420561] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.420568] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.420575] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.420581] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3708, tenant_id=1016, ls_id={id:1}) [2025-08-15 10:15:41.420590] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:41.421652] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:41.440983] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C686E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.442270] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.443429] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.444577] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.444598] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:41.444612] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.444621] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1001}, replicas:[{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:41.444734] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=87] build ls locality stat map(tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:15:41.444745] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.444756] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.444764] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.444773] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:41.444848] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.444860] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.444867] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.444871] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.444878] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3749, tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:15:41.444886] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:41.445964] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:41.456411] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7640-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.459572] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7640-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.460263] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224139326589005, replayable_scn:1755224139326589005, readable_scn:1755224139326589005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224141457076, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224139326589005, replayable_scn:1755224139326589005, readable_scn:1755224139326589005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224141457076, ora_rowscn_=1755224140813716000, content_changed=true) [2025-08-15 10:15:41.460445] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:41.460456] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:41.460462] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:41.461457] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7640-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.461555] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7640-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.464551] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E3E2-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}) [2025-08-15 10:15:41.464590] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E3E2-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:15:41.466669] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.467809] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.468955] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.468976] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:41.468990] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.468999] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1002}, replicas:[{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:41.469100] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=78] build ls locality stat map(tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:15:41.469107] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.469116] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=7] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.469124] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.469132] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:41.469207] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.469219] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.469226] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.469233] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.469239] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3697, tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:15:41.469247] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:41.470317] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:41.491159] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.492874] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.493034] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884A-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.494154] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A29-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.494594] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:41.494624] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:41.494647] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=15] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.494663] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=13] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1003}, replicas:[{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:41.494862] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=156] build ls locality stat map(tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:15:41.494881] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=11] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.494901] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=15] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.494919] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=14] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:41.494944] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=14] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:41.495113] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=129] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.495135] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=12] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.495152] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=12] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.495166] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=10] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:41.495180] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=11] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=5386, tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:15:41.495196] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=12] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:41.495564] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884A-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:41.495581] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.496634] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=10] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94839, tenant_id=1016, acc_dr_task=0) [2025-08-15 10:15:41.497445] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A29-0-0] [lt=91] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:41.497463] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A29-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.500734] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E724-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.500749] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0B5-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.532260] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224140005718004, replayable_scn:1755224140005718004, readable_scn:1755224140005718004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224141528691, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224140005718004, replayable_scn:1755224140005718004, readable_scn:1755224140005718004, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224141528691, ora_rowscn_=1755224141458129000, content_changed=true) [2025-08-15 10:15:41.532453] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:41.532466] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:41.532472] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:41.532838] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224139863844004, replayable_scn:1755224139863844004, readable_scn:1755224139863844004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224141529745, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224139863844004, replayable_scn:1755224139863844004, readable_scn:1755224139863844004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224141529745, ora_rowscn_=1755224141400304000, content_changed=true) [2025-08-15 10:15:41.533011] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:41.533022] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:41.533027] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:41.538450] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:41.538460] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:41.538465] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:41.544344] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224140005718004, replayable_scn:1755224140005718004, readable_scn:1755224139827164005, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224141541247, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224140005718004, replayable_scn:1755224140005718004, readable_scn:1755224139827164005, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224141541247, ora_rowscn_=1755224141399251000, content_changed=true) [2025-08-15 10:15:41.544502] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:41.544515] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:41.544523] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:41.544534] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1008, cost_time_us=167, return_code_array=[0, 0, 0], broadcast_times=2) [2025-08-15 10:15:41.549611] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224140327337001, replayable_scn:1755224140327337001, readable_scn:1755224140327337001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224141546517, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224140327337001, replayable_scn:1755224140327337001, readable_scn:1755224140327337001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224141546517, ora_rowscn_=1755224141399251000, content_changed=true) [2025-08-15 10:15:41.549785] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:41.549796] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:41.549801] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:41.689766] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6871-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.701096] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD83-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.706411] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=13] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224140327337001, replayable_scn:1755224140327337001, readable_scn:1755224140327337001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224141703688, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224140327337001, replayable_scn:1755224140327337001, readable_scn:1755224140327337001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224141703688, ora_rowscn_=1755224141399251000, content_changed=true) [2025-08-15 10:15:41.706611] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:41.706628] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:41.706638] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:41.719122] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC63C-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.737540] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C1-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:41.740065] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C1-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:41.750171] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF05C-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.750279] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD92-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.769376] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A3-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.771909] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A3-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:41.771927] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A3-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.776922] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709535][T1014_TimerWK2_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:41.787110] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:41.788561] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886F-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.790819] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7F9F-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224141790216000, start_id=1755224141790216000, end_id=1755224141790216001) [2025-08-15 10:15:41.791075] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886F-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:41.791093] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD886F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.791635] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7F9F-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224141790216000, v:0}, tenant_id=1014) [2025-08-15 10:15:41.818964] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8371-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.820036] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB36-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.822960] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8371-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.823336] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB36-0-0] [lt=98] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:41.823357] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB36-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:41.825000] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8371-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.825129] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8371-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:41.827078] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:41.828280] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224141827004, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:15:41.828302] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224146828301, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:15:41.847966] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D12-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.860072] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5258-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:41.860403] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E3-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:41.862539] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5258-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:41.862836] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E3-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:41.907967] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B36C-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:41.934304] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB868E-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:41.936781] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB868E-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:41.936855] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F7-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:41.939410] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F7-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:41.991343] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58DD-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:41.992049] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9C0-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.023748] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F9D-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:42.026270] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F9D-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:42.063537] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A1-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:42.066010] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A1-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:42.103726] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58D7-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.126817] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727134][T1016_TimerWK2_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:42.136056] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0B8-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.148442] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF059-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.185197] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D10-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.199846] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9C5-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.202643] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727959][T1018_TimerWK3_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=7] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:42.257020] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619486][T1012_TimerWK1_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=8] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:42.275869] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D4B-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89257812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}) [2025-08-15 10:15:42.275908] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D4B-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:15:42.304346] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A10-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.305625] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF68-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.306912] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A10-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:42.306930] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A10-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.308167] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF68-0-0] [lt=68] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:42.308185] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF68-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.309859] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2025-08-15 10:15:42.309879] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=16][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:42.315784] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:42.316955] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487604][T1004_TimerWK1_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=5] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:15:42.316981] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=1][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:42.331891] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:42.333123] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224142331817, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:15:42.333147] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224147333146, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:15:42.403691] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E3-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.403914] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7851-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.403975] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04A-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224142330300000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224142330300000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224142330300000, v:0}}], last_dump_ts_=1755224137372259, force_dump=false) [2025-08-15 10:15:42.404004] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04A-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.404124] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C18-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.406872] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E3-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.407059] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7851-0-0] [lt=19] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224142330300000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224142330300000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224142330300000, v:0}}], last_dump_ts_=1755224137375745, force_dump=false) [2025-08-15 10:15:42.407081] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7851-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.407233] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C18-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.407239] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04A-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.408975] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E3-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.409036] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7851-0-0] [lt=3] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.409184] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7851-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.409216] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E3-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.409222] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.409322] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.409326] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C18-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.409417] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C18-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.409504] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C2-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.412557] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C2-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.414493] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C2-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.414580] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C2-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.418208] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:42.419430] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224142418134, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:15:42.419454] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224147419453, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:15:42.441054] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C686F-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.460462] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:42.460473] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:42.460478] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:42.462541] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7641-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.466095] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7641-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.468172] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7641-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.468257] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7641-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.472767] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E8-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1013, thread_idx=0) [2025-08-15 10:15:42.474110] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1013, only_for_display=false) [2025-08-15 10:15:42.475372] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:42.475378] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728063][T1017_DRService][T1017][YB420ACB0122-000633A358855626-0-0] [lt=17] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1017, thread_idx=1) [2025-08-15 10:15:42.479036] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [728063][T1017_DRService][T1017][YB420ACB0122-000633A358855627-0-0] [lt=16] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2388, tenant_id=1017, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:42.481385] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [728063][T1017_DRService][T1017][YB420ACB0122-000633A358855627-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2328, tenant_id=1018, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:42.489313] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.490489] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.491637] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.491667] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1013, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:42.491688] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=13] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.491703] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1013, ls_id:{id:1}, replicas:[{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:42.491873] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=133] build ls locality stat map(tenant_id=1013, ls_id={id:1}) [2025-08-15 10:15:42.491891] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=11] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.491909] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=14] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.491923] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=11] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.491945] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=17] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:42.492089] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=109] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.492111] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=11] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.492125] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.492138] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=10] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.492151] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=10] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3997, tenant_id=1013, ls_id={id:1}) [2025-08-15 10:15:42.492166] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=12] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:42.493527] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=10] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19417, tenant_id=1013, acc_dr_task=0) [2025-08-15 10:15:42.494751] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1014, only_for_display=false) [2025-08-15 10:15:42.495947] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:42.498313] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884B-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.500143] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2A-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.500791] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0B6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.500808] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E725-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.500873] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884B-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:42.500891] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.502613] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2A-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:42.502631] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.507037] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF216-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.509786] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.510951] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.512098] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.512122] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:42.512139] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.512150] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1}, replicas:[{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:42.512252] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=79] build ls locality stat map(tenant_id=1014, ls_id={id:1}) [2025-08-15 10:15:42.512260] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.512274] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.512284] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.512295] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:42.512372] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.512386] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.512396] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.512401] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.512410] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3797, tenant_id=1014, ls_id={id:1}) [2025-08-15 10:15:42.512420] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:42.513490] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:42.525096] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A77-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.85742188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80273438GB}) [2025-08-15 10:15:42.525140] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A77-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:15:42.532475] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:42.532495] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:42.532501] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:42.533133] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:42.533148] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:42.533154] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:42.534444] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.534489] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:42.535609] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.536786] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.536810] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:42.536827] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.536839] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1001}, replicas:[{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:42.536950] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=87] build ls locality stat map(tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:15:42.536958] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.536972] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.536984] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.536997] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:42.537074] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.537088] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.537098] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.537104] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.537113] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3834, tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:15:42.537123] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:42.538233] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:42.538597] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:42.538609] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:42.538614] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:42.544630] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:42.544643] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:42.544649] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:42.545946] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:42.549887] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:42.549902] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:42.549907] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:42.559506] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.560694] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.561920] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.561948] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:42.561961] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.561970] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1002}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:42.562073] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=80] build ls locality stat map(tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:15:42.562083] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.562094] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.562102] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.562111] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:42.562186] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.562196] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.562203] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.562210] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.562217] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3877, tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:15:42.562225] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:42.563286] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:42.584271] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.585413] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.586559] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:42.586580] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:42.586595] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.586604] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1003}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:42.586712] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=85] build ls locality stat map(tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:15:42.586722] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.586733] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.586741] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:42.586749] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:42.586831] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=59] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.586843] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.586856] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.586863] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:42.586869] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3738, tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:15:42.586878] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:42.587981] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94436, tenant_id=1014, acc_dr_task=0) [2025-08-15 10:15:42.689842] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6872-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.701175] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD84-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.706743] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:42.706755] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:42.706761] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:42.745572] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C2-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:42.748084] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C2-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:42.750243] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF05D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.750348] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD93-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.774605] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A4-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.777123] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A4-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:42.777140] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A4-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.793761] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8870-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.796328] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8870-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:42.796345] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8870-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.796556] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:42.797778] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224142796481, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:15:42.797803] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224147797802, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:15:42.797964] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:42.799173] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224142797891, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:15:42.799193] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224147799192, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:15:42.826175] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8372-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.826191] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB37-0-0] [lt=47] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.829669] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB37-0-0] [lt=83] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:42.829687] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB37-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:42.829776] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:340) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8372-0-0] [lt=17] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1014, ls_id:{id:1}, sync_scn:{val:1755224142330300005, v:0}, readable_scn:{val:1755224142330300005, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-08-15 10:15:42.830162] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8372-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.832114] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8372-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.832216] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8372-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:42.832256] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:459) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8372-0-0] [lt=7] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1755224140327337005, v:0}) [2025-08-15 10:15:42.848037] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D13-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.867873] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5259-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:42.868091] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E4-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:42.870602] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E4-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:42.871604] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5259-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:42.909410] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B36D-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:42.914597] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=12] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224142536376, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224141248886000, replayable_scn:1755224141248886000, readable_scn:1755224141248886000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:42.942115] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB868F-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:42.944627] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F8-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:42.944644] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB868F-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:42.947108] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F8-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:42.959923] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:42.961167] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224142959850, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:15:42.961201] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=25] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224147961199, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:15:42.991385] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58DE-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:42.992125] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9C1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.031725] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F9E-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:43.035542] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F9E-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:43.045736] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:43.046984] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224143045662, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:15:43.047008] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224148047005, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:15:43.071374] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A2-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:43.073840] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A2-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:43.103794] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58D8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.136133] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0B9-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.148514] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF05A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.185270] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D11-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.199920] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9C6-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.309693] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A11-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.310904] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF69-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.312261] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A11-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:43.312280] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A11-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.313424] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF69-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:43.313442] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF69-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.321469] INFO [STORAGE] check_snapshot_readable_with_cache (ob_tablet.cpp:8380) [728060][T1017_BackupCle][T1017][YB420ACB0122-000633A355955647-0-0] [lt=18] refresh ddl data cache(ret=0, ls_id={id:1}, tablet_id={id:404}, ddl_data_cache_={redefined:false, schema_version:0, snapshot_version:0}) [2025-08-15 10:15:43.321517] WDIAG [SHARE] get_default_delete_policy (ob_backup_clean_operator.cpp:1397) [728060][T1017_BackupCle][T1017][YB420ACB0122-000633A355955647-0-0] [lt=20][errcode=-4018] entry not exit(ret=-4018, sql=select * from __all_backup_delete_policy where tenant_id=1018 and policy_name='default') [2025-08-15 10:15:43.321597] INFO [RS] start_auto_delete_obsolete_data_ (ob_backup_clean_scheduler.cpp:1959) [728060][T1017_BackupCle][T1017][YB420ACB0122-000633A355955647-0-0] [lt=10] [BACKUP_CLEAN] finish schedule auto delete(tenant_ids=[]) [2025-08-15 10:15:43.321609] WDIAG [RS] process (ob_backup_clean_scheduler.cpp:1895) [728060][T1017_BackupCle][T1017][YB420ACB0122-000633A355955647-0-0] [lt=9][errcode=-4018] failed to start auto obsolete delete(ret=-4018) [2025-08-15 10:15:43.321620] WDIAG [RS] process_trigger_ (ob_backup_service.cpp:454) [728060][T1017_BackupCle][T1017][YB420ACB0122-000633A355955647-0-0] [lt=8][errcode=-4018] job status move forward failed(*trigger={trigger_type:0}) [2025-08-15 10:15:43.410189] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7852-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.410378] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.410387] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C19-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.410413] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E4-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224143332205001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224143332205001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224143332205001, v:0}}], last_dump_ts_=1755224138378730, force_dump=false) [2025-08-15 10:15:43.410438] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E4-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.413208] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7852-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.413266] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E4-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224143332205001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224143332205001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224143332205001, v:0}}], last_dump_ts_=1755224138381934, force_dump=false) [2025-08-15 10:15:43.413291] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E4-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.413432] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04B-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.413461] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C19-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.415159] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7852-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.415348] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7852-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.415356] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E4-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.415448] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.415458] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E4-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.415533] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C19-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.415546] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04B-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.415616] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C19-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.415641] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C3-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.418400] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C3-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.420374] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C3-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.420456] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C3-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.441082] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6870-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.460553] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:43.460564] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:43.460572] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:43.464994] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A0888-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}) [2025-08-15 10:15:43.465035] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A0888-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:43.469266] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7642-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.472253] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7642-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.474145] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7642-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.474252] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7642-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.500866] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0B7-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.500883] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E726-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.503072] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884C-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.505314] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2B-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.505683] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884C-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:43.505699] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884C-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.507848] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2B-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:43.507863] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2B-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.531767] INFO [RS] gc_mview (ob_mview_maintenance_task.cpp:267) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=16] mview maintenance task gc mview success(tenant_id_=1012, round_=77507, gc_mview_cost_us_=1232, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:15:43.532520] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:43.532532] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:43.532540] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:43.533119] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:43.533132] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:43.533140] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:43.538892] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224142331355002, replayable_scn:1755224142331355002, readable_scn:1755224142331355002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224143536293, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224142331355002, replayable_scn:1755224142331355002, readable_scn:1755224142331355002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224143536293, ora_rowscn_=1755224143412163000, content_changed=true) [2025-08-15 10:15:43.539062] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:43.539075] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:43.539080] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:43.540347] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:43.544597] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:43.544610] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:43.544617] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:43.549982] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:43.549993] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:43.550000] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:43.603717] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:43.604960] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224143603646, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:15:43.604983] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224148604982, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:15:43.689923] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6873-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.699664] INFO [RS] prepare (ob_mview_maintenance_task.cpp:176) [709535][T1014_TimerWK2_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=16] mview maintenance task prepare success(tenant_id_=1014, round_=63450, prepare_cost_us_=0) [2025-08-15 10:15:43.701252] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD85-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.706834] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:43.706848] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:43.706855] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:43.724068] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC63D-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.750313] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF05E-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.750417] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD94-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.753310] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C3-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:43.755790] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C3-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:43.779877] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A5-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.782397] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A5-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:43.782414] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A5-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.799024] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8871-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.801550] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8871-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:43.801566] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8871-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.832489] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB38-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.833179] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8373-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.835095] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB38-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:43.835113] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB38-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:43.836271] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8373-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224143332205005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224143332205005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224143332205004, v:0}}], last_dump_ts_=1755224138800842, force_dump=false) [2025-08-15 10:15:43.836295] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8373-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.838194] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8373-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.838290] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8373-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:43.848110] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D14-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.875892] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E5-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:43.876784] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD525A-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:43.878414] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E5-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:43.879233] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD525A-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:43.910831] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B36E-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:43.949913] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8690-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:43.952357] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F9-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:43.952389] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8690-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:43.955987] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52F9-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:43.991457] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58DF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:43.992178] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9C2-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.041091] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F9F-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:44.043666] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5F9F-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:44.079337] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A3-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:44.083008] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A3-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:44.103873] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58D9-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.136205] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0BA-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.148585] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF05B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.185339] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D12-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.199992] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9C7-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.276371] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B634B-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.89648438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}) [2025-08-15 10:15:44.276409] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B634B-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:15:44.315067] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A12-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.316160] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6A-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.317596] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A12-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:44.317615] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A12-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.318679] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6A-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:44.318697] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.322957] INFO [RS.LB] idle (ob_root_balancer.cpp:87) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=29] root balance idle(idle_time=10000000) [2025-08-15 10:15:44.322974] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:70) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=12] start check unit migrate finish [2025-08-15 10:15:44.322986] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=5] try check migrate unit finish by tenant(tenant_id=1) [2025-08-15 10:15:44.332375] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1004) [2025-08-15 10:15:44.358295] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1006) [2025-08-15 10:15:44.384224] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1008) [2025-08-15 10:15:44.410031] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1012) [2025-08-15 10:15:44.416350] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7853-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224144332960001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224144332960001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224144332960001, v:0}}], last_dump_ts_=1755224139385079, force_dump=false) [2025-08-15 10:15:44.416377] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7853-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.416630] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.416705] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E5-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.416892] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1A-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224144253049000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224144253049000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224144253049000, v:0}}], last_dump_ts_=1755224139385181, force_dump=false) [2025-08-15 10:15:44.416918] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1A-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.419369] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7853-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.419743] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E5-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.419762] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04C-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.419896] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1A-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224144332960001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224144332960001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224144332960001, v:0}}], last_dump_ts_=1755224139388526, force_dump=false) [2025-08-15 10:15:44.419918] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1A-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.421370] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7853-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.421420] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C4-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.421474] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7853-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.421719] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.421722] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E5-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.421801] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E5-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.421816] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.421864] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1A-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.421957] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1A-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.424337] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C4-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.426306] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C4-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.426408] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C4-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.436008] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1014) [2025-08-15 10:15:44.441157] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6871-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.460434] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224142330300005, replayable_scn:1755224142330300005, readable_scn:1755224142330300005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224144457136, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224142330300005, replayable_scn:1755224142330300005, readable_scn:1755224142330300005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224144457136, ora_rowscn_=1755224143834814000, content_changed=true) [2025-08-15 10:15:44.460610] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:44.460621] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:44.460626] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:44.461756] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1016) [2025-08-15 10:15:44.461943] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:44.475241] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7643-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224144334012000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224144452921001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224144452921001, v:0}}], last_dump_ts_=1755224139444316, force_dump=false) [2025-08-15 10:15:44.475270] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7643-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.478266] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7643-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224144354028000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224144452921000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224144452921000, v:0}}], last_dump_ts_=1755224139447446, force_dump=false) [2025-08-15 10:15:44.478289] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7643-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.480298] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7643-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.480381] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7643-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.487441] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1018) [2025-08-15 10:15:44.500942] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0B8-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.500956] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E727-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.507309] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF217-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.507853] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884D-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.511677] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884D-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:44.511697] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884D-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.511677] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2C-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.513198] INFO [RS] check (ob_alter_locality_finish_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814F9-0-0] [lt=13] start to check alter locality finish [2025-08-15 10:15:44.514278] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2C-0-0] [lt=78] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:44.514294] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2C-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.514394] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814FA-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.515572] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814FA-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.517860] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814FC-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.519012] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814FC-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.521300] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814FE-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.522470] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A814FE-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.524763] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81500-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.525584] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBA8B-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80273438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}) [2025-08-15 10:15:44.525625] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBA8B-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2025-08-15 10:15:44.525906] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81500-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.528193] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81502-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.529332] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81502-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.531631] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81504-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.532408] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224143452179000, replayable_scn:1755224143452179000, readable_scn:1755224143452179000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224144529525, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224143452179000, replayable_scn:1755224143452179000, readable_scn:1755224143452179000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224144529525, ora_rowscn_=1755224144477045000, content_changed=true) [2025-08-15 10:15:44.532609] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:44.532620] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:44.532628] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:44.532823] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81504-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.533049] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224143332205001, replayable_scn:1755224143332205001, readable_scn:1755224143332205001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224144529525, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224143332205001, replayable_scn:1755224143332205001, readable_scn:1755224143332205001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224144529525, ora_rowscn_=1755224144423466000, content_changed=true) [2025-08-15 10:15:44.533243] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:44.533257] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:44.533262] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:44.533902] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:44.535117] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81506-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.536262] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81506-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.538550] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81508-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.539073] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:44.539087] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:44.539094] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:44.539725] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81508-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.540928] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81508-0-0] [lt=12] start check shrink resource pool [2025-08-15 10:15:44.540952] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81509-0-0] [lt=16] start check shrink resource pool(tenant_id=1) [2025-08-15 10:15:44.542067] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81509-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.543213] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81509-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.543281] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150A-0-0] [lt=14] start check shrink resource pool(tenant_id=1004) [2025-08-15 10:15:44.544413] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150A-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.544560] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224143332205001, replayable_scn:1755224143332205001, readable_scn:1755224143332205001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224144541110, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224143332205001, replayable_scn:1755224143332205001, readable_scn:1755224143332205001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224144541110, ora_rowscn_=1755224144418197000, content_changed=true) [2025-08-15 10:15:44.544739] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:44.544751] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:44.544756] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:44.545592] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150A-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.545645] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150B-0-0] [lt=12] start check shrink resource pool(tenant_id=1006) [2025-08-15 10:15:44.546769] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150B-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.547930] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150B-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.547996] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150C-0-0] [lt=17] start check shrink resource pool(tenant_id=1008) [2025-08-15 10:15:44.549147] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150C-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.549905] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224143332205001, replayable_scn:1755224143332205001, readable_scn:1755224143332205001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224144546385, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224143332205001, replayable_scn:1755224143332205001, readable_scn:1755224143332205001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224144546385, ora_rowscn_=1755224144418197000, content_changed=true) [2025-08-15 10:15:44.550091] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:44.550110] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:44.550118] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:44.550132] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1006, cost_time_us=205, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:44.550309] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150C-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.550363] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150D-0-0] [lt=12] start check shrink resource pool(tenant_id=1012) [2025-08-15 10:15:44.551484] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150D-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.551496] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:44.552628] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150D-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.552679] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150E-0-0] [lt=12] start check shrink resource pool(tenant_id=1014) [2025-08-15 10:15:44.553794] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150E-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.554935] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150E-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.554994] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150F-0-0] [lt=17] start check shrink resource pool(tenant_id=1016) [2025-08-15 10:15:44.556103] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150F-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.557242] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8150F-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.557295] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81510-0-0] [lt=13] start check shrink resource pool(tenant_id=1018) [2025-08-15 10:15:44.558406] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81510-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.559609] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81510-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.559669] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:99) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81510-0-0] [lt=13] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.559684] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81510-0-0] [lt=11] start check primary zone [2025-08-15 10:15:44.559692] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81511-0-0] [lt=4] start check primary zone(tenant_id=1) [2025-08-15 10:15:44.560806] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81511-0-0] [lt=6][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.560859] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81512-0-0] [lt=13] start check primary zone(tenant_id=1004) [2025-08-15 10:15:44.562010] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81512-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.562067] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81513-0-0] [lt=12] start check primary zone(tenant_id=1006) [2025-08-15 10:15:44.563207] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81513-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.563265] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81514-0-0] [lt=13] start check primary zone(tenant_id=1008) [2025-08-15 10:15:44.564377] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81514-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.564429] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81515-0-0] [lt=12] start check primary zone(tenant_id=1012) [2025-08-15 10:15:44.565539] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81515-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.565592] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81516-0-0] [lt=13] start check primary zone(tenant_id=1014) [2025-08-15 10:15:44.566697] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81516-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.566760] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81517-0-0] [lt=11] start check primary zone(tenant_id=1016) [2025-08-15 10:15:44.567871] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81517-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.567923] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=12] start check primary zone(tenant_id=1018) [2025-08-15 10:15:44.569034] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:44.569090] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:166) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=15] start do unit balance [2025-08-15 10:15:44.569872] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:104) [17458][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=10] start checking lost replicas [2025-08-15 10:15:44.575319] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:350) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=9] start distribute for server status change [2025-08-15 10:15:44.575415] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:401) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=11] finish distribute for server status change(ret=0) [2025-08-15 10:15:44.575425] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:200) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=8] finish do unit balance(ret=0) [2025-08-15 10:15:44.581352] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:115) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=5] start to do tenant group unit balance [2025-08-15 10:15:44.581393] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=11] start to do tenant group unit balance(zone=zone1) [2025-08-15 10:15:44.581406] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=8] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.203.1.34:2882", disk_in_use_:3126853632, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:44.581423] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=11] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:44.581451] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=10] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:44.581466] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=6] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:44.581474] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone1) [2025-08-15 10:15:44.581483] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=5] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:44.581502] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=11] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.1.34:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:44.581542] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=32] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.203.1.34:2882", disk_in_use_:3126853632, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:44.581550] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=7] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.1.34:2882", disk_in_use_:3126853632, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:44.581562] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=7] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:44.581574] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=9] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1006, unit_config:{unit_config_id:1005, name:"config_opsone_zone1_s_2_4_ywx", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7988, unit_config:0x7f1e9aa71c38, pool:0x7f1c3beb2f18}, {unit_id:1008, tenant_id:1006, resource_pool_id:1008, unit_config:{unit_config_id:1008, name:"config_test_zone1_s_2_4_emp", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f77d8, unit_config:0x7f1e9aa71998, pool:0x7f1c3beb27a8}, {unit_id:1011, tenant_id:1008, resource_pool_id:1011, unit_config:{unit_config_id:1011, name:"config_mcp_zone1_s_4_8_tmc", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7550, unit_config:0x7f1e9aa716f8, pool:0x7f1e443fda40}, {unit_id:1016, tenant_id:1012, resource_pool_id:1016, unit_config:{unit_config_id:1017, name:"config_oms_zone1_s_4_8_gzp", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f73a0, unit_config:0x7f1e9aa71458, pool:0x7f1e443fd2d0}, {unit_id:1019, tenant_id:1014, resource_pool_id:1019, unit_config:{unit_config_id:1019, name:"config_grafana_test_zone1_s_1_2_pri", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7118, unit_config:0x7f1e9aa71298, pool:0x7f1e443fc7a8}, {unit_id:1024, tenant_id:1016, resource_pool_id:1024, unit_config:{unit_config_id:1023, name:"config_temporalio_zone1_s_2_4_ykw", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6ce0, unit_config:0x7f1e9aa70f18, pool:0x7f1e74dff2d0}, {unit_id:1026, tenant_id:1018, resource_pool_id:1026, unit_config:{unit_config_id:1027, name:"config_media_zone1_s_2_4_kmn", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6b30, unit_config:0x7f1e9aa70b98, pool:0x7f1e74dfeb60}], not_grant_units=[], available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:44.581630] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=33] start to do tenant group unit balance(zone=zone2) [2025-08-15 10:15:44.581638] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=5] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.203.4.38:2882", disk_in_use_:3114270720, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:44.581648] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=7] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:44.581663] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=8] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:44.581673] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=5] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:44.581679] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=5] start do non-tenantgroup unit balance task(zone=zone2) [2025-08-15 10:15:44.581686] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=4] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:44.581697] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=7] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.38:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:44.581728] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=24] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.203.4.38:2882", disk_in_use_:3114270720, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:44.581735] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=6] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.38:2882", disk_in_use_:3114270720, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:44.581744] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=6] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:44.581752] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=6] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1004, unit_config:{unit_config_id:1004, name:"config_opsone_zone2_s_2_4_wag", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7b38, unit_config:0x7f1e9aa71d18, pool:0x7f1c3beb3688}, {unit_id:1009, tenant_id:1006, resource_pool_id:1009, unit_config:{unit_config_id:1009, name:"config_test_zone2_s_2_4_yey", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7700, unit_config:0x7f1e9aa718b8, pool:0x7f1c3beb23f0}, {unit_id:1010, tenant_id:1008, resource_pool_id:1010, unit_config:{unit_config_id:1010, name:"config_mcp_zone2_s_4_8_bjb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7628, unit_config:0x7f1e9aa717d8, pool:0x7f1c3beb2038}, {unit_id:1018, tenant_id:1012, resource_pool_id:1018, unit_config:{unit_config_id:1016, name:"config_oms_zone2_s_4_8_kbx", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f71f0, unit_config:0x7f1e9aa71538, pool:0x7f1e443fcb60}, {unit_id:1021, tenant_id:1014, resource_pool_id:1021, unit_config:{unit_config_id:1021, name:"config_grafana_test_zone2_s_1_2_yyc", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f6f68, unit_config:0x7f1e9aa710d8, pool:0x7f1e443fc038}, {unit_id:1023, tenant_id:1016, resource_pool_id:1023, unit_config:{unit_config_id:1022, name:"config_temporalio_zone2_s_2_4_lsv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6db8, unit_config:0x7f1e9aa70ff8, pool:0x7f1e74dff688}, {unit_id:1027, tenant_id:1018, resource_pool_id:1027, unit_config:{unit_config_id:1025, name:"config_media_zone2_s_2_4_yzd", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6a58, unit_config:0x7f1e9aa70d58, pool:0x7f1e74dfe7a8}], not_grant_units=[], available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:44.581801] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=28] start to do tenant group unit balance(zone=zone3) [2025-08-15 10:15:44.581808] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=5] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.203.4.39:2882", disk_in_use_:3013607424, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:44.581818] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=7] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:44.581832] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=7] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:44.581841] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=5] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:44.581848] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=5] start do non-tenantgroup unit balance task(zone=zone3) [2025-08-15 10:15:44.581855] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=5] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:44.581866] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=7] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.39:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:44.581898] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=25] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.203.4.39:2882", disk_in_use_:3013607424, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:44.581904] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=4] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.39:2882", disk_in_use_:3013607424, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:44.581913] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=7] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:44.581921] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=6] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1005, unit_config:{unit_config_id:1006, name:"config_opsone_zone3_s_2_4_vxv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7a60, unit_config:0x7f1e9aa71b58, pool:0x7f1c3beb32d0}, {unit_id:1007, tenant_id:1006, resource_pool_id:1007, unit_config:{unit_config_id:1007, name:"config_test_zone3_s_2_4_mui", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f78b0, unit_config:0x7f1e9aa71a78, pool:0x7f1c3beb2b60}, {unit_id:1012, tenant_id:1008, resource_pool_id:1012, unit_config:{unit_config_id:1012, name:"config_mcp_zone3_s_4_8_seu", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7478, unit_config:0x7f1e9aa71618, pool:0x7f1e443fd688}, {unit_id:1017, tenant_id:1012, resource_pool_id:1017, unit_config:{unit_config_id:1018, name:"config_oms_zone3_s_4_8_nvb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f72c8, unit_config:0x7f1e9aa71378, pool:0x7f1e443fcf18}, {unit_id:1020, tenant_id:1014, resource_pool_id:1020, unit_config:{unit_config_id:1020, name:"config_grafana_test_zone3_s_1_2_xbz", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7040, unit_config:0x7f1e9aa711b8, pool:0x7f1e443fc3f0}, {unit_id:1022, tenant_id:1016, resource_pool_id:1022, unit_config:{unit_config_id:1024, name:"config_temporalio_zone3_s_2_4_rxz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6e90, unit_config:0x7f1e9aa70e38, pool:0x7f1e74dffa40}, {unit_id:1025, tenant_id:1018, resource_pool_id:1025, unit_config:{unit_config_id:1026, name:"config_media_zone3_s_2_4_ukz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6c08, unit_config:0x7f1e9aa70c78, pool:0x7f1e74dfef18}], not_grant_units=[], available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:44.624458] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:44.625684] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224144624386, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:15:44.625708] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224149625706, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:15:44.690012] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6874-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.701329] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD86-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.706584] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224143332205001, replayable_scn:1755224143332205001, readable_scn:1755224143332205001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224144702945, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224143332205001, replayable_scn:1755224143332205001, readable_scn:1755224143332205001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224144702945, ora_rowscn_=1755224144418197000, content_changed=true) [2025-08-15 10:15:44.706760] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:44.706771] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:44.706779] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:44.708074] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:44.721619] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476036][T1007_DRService][T1007][YB420ACB0122-0006338215870D0E-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1007, thread_idx=1) [2025-08-15 10:15:44.725296] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [476036][T1007_DRService][T1007][YB420ACB0122-0006338215870D0F-0-0] [lt=11] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2362, tenant_id=1007, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:44.727648] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [476036][T1007_DRService][T1007][YB420ACB0122-0006338215870D0F-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2331, tenant_id=1008, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:44.750384] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF05F-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.750487] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD95-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.761104] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C4-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:44.763577] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C4-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:44.785139] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A6-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.787636] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A6-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:44.787656] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A6-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.804221] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8872-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.806724] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8872-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:44.806743] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8872-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.837873] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB39-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.839307] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8374-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224144332960005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224144332960005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224144332960005, v:0}}], last_dump_ts_=1755224139803958, force_dump=false) [2025-08-15 10:15:44.839334] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8374-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.840422] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB39-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:44.840439] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB39-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:44.842352] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8374-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.844306] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8374-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.844536] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8374-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:44.848182] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D15-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.876714] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C56-0-0] [lt=15] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1011, thread_idx=0) [2025-08-15 10:15:44.878004] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1011, only_for_display=false) [2025-08-15 10:15:44.879253] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:44.884346] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E6-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:44.884637] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD525B-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:44.886891] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E6-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:44.887227] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD525B-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:44.894630] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.895790] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.896927] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.896956] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1011, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:44.896974] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.896987] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1011, ls_id:{id:1}, replicas:[{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:44.897097] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=85] build ls locality stat map(tenant_id=1011, ls_id={id:1}) [2025-08-15 10:15:44.897106] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.897120] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.897131] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.897142] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:44.897221] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.897235] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.897245] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.897251] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.897260] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3783, tenant_id=1011, ls_id={id:1}) [2025-08-15 10:15:44.897271] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:44.898741] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=20738, tenant_id=1011, acc_dr_task=0) [2025-08-15 10:15:44.899979] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1012, only_for_display=false) [2025-08-15 10:15:44.901184] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:44.912249] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B36F-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:44.916911] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.918074] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.919217] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.919239] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:44.919257] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.919269] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1}, replicas:[{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:44.919379] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=85] build ls locality stat map(tenant_id=1012, ls_id={id:1}) [2025-08-15 10:15:44.919388] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.919401] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.919413] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.919421] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:44.919508] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=63] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.919523] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.919534] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.919541] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.919549] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3792, tenant_id=1012, ls_id={id:1}) [2025-08-15 10:15:44.919560] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:44.921153] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:44.943326] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.944477] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.945630] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.945655] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:44.945673] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.945685] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1001}, replicas:[{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:44.945798] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=87] build ls locality stat map(tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:15:44.945807] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.945820] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.945831] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.945844] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:44.945924] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.945943] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.945953] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.945959] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.945965] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3782, tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:15:44.945976] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:44.947378] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:44.957663] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8691-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:44.960250] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8691-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:44.961251] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52FA-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:44.963739] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52FA-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:44.969655] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.970812] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.971982] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.972007] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:44.972021] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.972031] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1002}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:44.972141] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=86] build ls locality stat map(tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:15:44.972150] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.972164] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.972171] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.972181] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:44.972261] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.972276] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.972285] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.972292] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.972302] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3786, tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:15:44.972313] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:44.973806] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:44.991527] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58E0-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.992250] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9C3-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:44.995958] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.997104] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.998245] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:44.998268] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:44.998287] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.998297] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1003}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:44.998409] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=87] build ls locality stat map(tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:15:44.998420] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.998432] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.998441] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:44.998451] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:44.998528] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.998541] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.998550] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.998557] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:44.998565] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3757, tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:15:44.998574] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:45.000149] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=101392, tenant_id=1012, acc_dr_task=0) [2025-08-15 10:15:45.001055] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17740][T1_TimerWK3_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:45.049056] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA0-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:45.051639] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA0-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:45.088561] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A4-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:45.091110] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A4-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:45.103931] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58DA-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.124108] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [1487708][T1003_ArchiveSv][T1003][YB420ACB0122-0006337FE36598F1-0-0] [lt=7] start do ObArchiveSchedulerService round(round=186150) [2025-08-15 10:15:45.127639] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [1487708][T1003_ArchiveSv][T1003][YB420ACB0122-0006337FE36598F1-0-0] [lt=13][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:15:45.127655] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [1487708][T1003_ArchiveSv][T1003][YB420ACB0122-0006337FE36598F1-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1004 AND dest_no = 0 AND name = 'state', exec_tenant_id=1003) [2025-08-15 10:15:45.127699] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [1487708][T1003_ArchiveSv][T1003][YB420ACB0122-0006337FE36598F1-0-0] [lt=10][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1004, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:45.127715] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [1487708][T1003_ArchiveSv][T1003][YB420ACB0122-0006337FE36598F1-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1004, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:15:45.127727] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [1487708][T1003_ArchiveSv][T1003][YB420ACB0122-0006337FE36598F1-0-0] [lt=9][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1004, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:45.136274] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0BB-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.148662] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF05C-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.185418] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D13-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.200066] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9C8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.215270] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED4-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1007, thread_idx=0) [2025-08-15 10:15:45.216575] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1007, only_for_display=false) [2025-08-15 10:15:45.217831] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:45.232719] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.233899] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.235055] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.235077] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1007, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:45.235092] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.235101] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1007, ls_id:{id:1}, replicas:[{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:45.235202] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=79] build ls locality stat map(tenant_id=1007, ls_id={id:1}) [2025-08-15 10:15:45.235210] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.235221] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.235230] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.235239] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:45.235335] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=71] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.235347] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.235354] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.235362] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.235369] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3818, tenant_id=1007, ls_id={id:1}) [2025-08-15 10:15:45.235377] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:45.236451] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19876, tenant_id=1007, acc_dr_task=0) [2025-08-15 10:15:45.237679] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1008, only_for_display=false) [2025-08-15 10:15:45.238911] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:45.252869] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.254031] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.255191] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.255214] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:45.255228] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.255237] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1}, replicas:[{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:45.255338] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=79] build ls locality stat map(tenant_id=1008, ls_id={id:1}) [2025-08-15 10:15:45.255346] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.255357] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.255365] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.255374] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:45.255450] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.255465] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.255475] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.255484] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.255494] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3791, tenant_id=1008, ls_id={id:1}) [2025-08-15 10:15:45.255506] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:45.256595] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:45.277485] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.278652] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.279814] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.279837] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:45.279854] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.279866] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1001}, replicas:[{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:45.279977] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=87] build ls locality stat map(tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:15:45.279985] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.279998] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.280005] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.280017] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:45.280103] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=63] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.280113] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.280123] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.280132] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.280138] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3805, tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:15:45.280149] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:45.281262] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:45.302243] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.303418] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.304581] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.304606] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:45.304623] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.304634] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1002}, replicas:[{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:45.304745] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=86] build ls locality stat map(tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:15:45.304754] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.304767] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.304779] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.304786] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:45.304865] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.304881] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.304890] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.304899] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.304909] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3844, tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:15:45.304920] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:45.306109] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:45.320357] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A13-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.321278] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227745320089}) [2025-08-15 10:15:45.321295] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:45.321308] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:15:45.321372] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6B-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.322931] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A13-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:45.322958] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A13-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.323926] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6B-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:45.323949] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6B-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.327030] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.327193] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227745326019}) [2025-08-15 10:15:45.327213] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:45.327230] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:15:45.328191] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.328425] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227745327248}) [2025-08-15 10:15:45.328440] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:45.328452] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:15:45.329345] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:45.329368] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:45.329384] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.329396] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1003}, replicas:[{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:45.329499] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=80] build ls locality stat map(tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:15:45.329507] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.329519] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.329531] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:45.329543] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:45.329620] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.329634] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.329644] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.329651] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:45.329660] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3786, tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:15:45.329669] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:45.330737] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94272, tenant_id=1008, acc_dr_task=0) [2025-08-15 10:15:45.422476] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7854-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.422848] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.422848] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E6-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.423295] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1B-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.425608] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7854-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.426126] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.426311] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1B-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.426441] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E6-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.427350] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C5-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.427527] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7854-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.427622] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7854-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.428188] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04D-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.428293] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.428407] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.428492] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.429006] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E6-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.429120] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E6-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.430322] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C5-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.432293] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C5-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.432384] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C5-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.441233] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6872-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.460949] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:45.460962] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:45.460967] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:45.465535] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A0904-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}) [2025-08-15 10:15:45.465588] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A0904-0-0] [lt=33] [DDL] execute ddl like stmt(ret=0, cost=57, ddl_arg=NULL) [2025-08-15 10:15:45.481467] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7644-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.484575] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7644-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.486729] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7644-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.486817] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7644-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.501015] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0B9-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.501032] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E728-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.513889] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884E-0-0] [lt=44] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.516523] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884E-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:45.516541] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884E-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.516963] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2D-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.519767] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2D-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:45.519791] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2D-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.532513] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:45.532525] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:45.532533] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:45.533225] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:45.533239] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:45.533246] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:45.539117] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:45.539133] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:45.539155] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:45.545382] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:45.545393] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:45.545399] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:45.550052] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:45.550065] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:45.550071] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:45.619423] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:45.620651] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224145619351, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:15:45.620676] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224150620674, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:15:45.689785] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438023][T1006_TimerWK2_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=8] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:45.690087] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6875-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.701407] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD87-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.706828] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:45.706839] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:45.706844] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:45.728962] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC63E-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.750458] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF060-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.750560] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD96-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.768972] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C5-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:45.771476] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C5-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:45.790399] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A7-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.792908] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A7-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:45.792927] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A7-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.809447] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8873-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.811980] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8873-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:45.811996] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8873-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.843220] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3A-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.845555] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8375-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.845731] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3A-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:45.845749] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:45.848258] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D16-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.848717] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8375-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.850879] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8375-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.850975] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8375-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:45.892247] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E7-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:45.892506] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD525C-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:45.894751] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E7-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:45.894964] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD525C-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:45.913610] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B370-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:45.916056] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=12] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224145535845, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224144334012001, replayable_scn:1755224144334012001, readable_scn:1755224144334012001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:45.944079] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:45.945297] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224145944005, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:15:45.945321] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224150945320, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:15:45.965531] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8692-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:45.968032] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8692-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:45.969056] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52FB-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:45.971615] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52FB-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:45.991600] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58E1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.992324] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9C4-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:45.995570] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:45.996884] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224145986716, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:15:45.996916] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224150996914, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:15:45.996948] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224150996936, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:46.005399] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2845) [17349][TimerWK2_HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=20] need remove task(remove_task_ids=[]) [2025-08-15 10:15:46.005417] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2864) [17349][TimerWK2_HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=14] remove all timeout ddl task succeed [2025-08-15 10:15:46.057169] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA1-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:46.059759] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA1-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:46.096588] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A5-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:46.099281] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A5-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:46.104012] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58DB-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.116818] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [2619609][T1011_BackupDat][T1011][YB420ACB0122-0006338CC14639D1-0-0] [lt=13] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:46.121926] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [2619608][T1011_BACKUP_SC][T1011][YB420ACB0122-0006338CC132C1CC-0-0] [lt=14] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:15:46.136348] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0BC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.148738] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF05D-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.185486] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D14-0-0] [lt=9] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.186898] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:46.188140] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224146186825, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:15:46.188162] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224151188160, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:15:46.189489] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:46.190665] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224146189421, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:15:46.190686] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224151190685, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:46.200135] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9C9-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.265574] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [1487706][T1003_BackupDat][T1003][YB420ACB0122-0006337FE347F6D7-0-0] [lt=13] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:46.276902] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D4D-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90039062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}) [2025-08-15 10:15:46.276949] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D4D-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=50, ddl_arg=NULL) [2025-08-15 10:15:46.292633] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:232) [1487705][T1003_BACKUP_SC][T1003][YB420ACB0122-0006337FE332C1CC-0-0] [lt=11] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2025-08-15 10:15:46.325712] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A14-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.326576] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6C-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.328276] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A14-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:46.328295] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A14-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.329093] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6C-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:46.329113] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6C-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.428986] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7855-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.429219] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.429452] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.429996] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E7-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.432097] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7855-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.432347] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1C-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.432346] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04E-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224146335837001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224146335837001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224146335837001, v:0}}], last_dump_ts_=1755224141400875, force_dump=false) [2025-08-15 10:15:46.432365] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.432956] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E7-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.433299] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C6-0-0] [lt=5] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224146335837001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224146335837001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224146335837001, v:0}}], last_dump_ts_=1755224141397630, force_dump=false) [2025-08-15 10:15:46.433326] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C6-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.434244] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7855-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.434387] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.434390] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04E-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.434416] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7855-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.434481] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.434495] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.434858] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E7-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.434932] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E7-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.436304] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C6-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224146335837001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224146335837001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224146335837001, v:0}}], last_dump_ts_=1755224141401415, force_dump=false) [2025-08-15 10:15:46.436329] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C6-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.438230] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C6-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.438308] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C6-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.441306] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6873-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.460657] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:46.460669] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:46.460674] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:46.487762] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7645-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.490652] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7645-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.492528] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7645-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.492634] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7645-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.501092] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0BA-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.501109] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E729-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.507586] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF218-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.518752] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884F-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.521295] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884F-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:46.521313] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D884F-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.522485] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2E-0-0] [lt=46] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.525039] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2E-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:46.525057] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2E-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.526075] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5818-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.80664062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81054688GB}) [2025-08-15 10:15:46.526119] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341EA78D5818-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=48, ddl_arg=NULL) [2025-08-15 10:15:46.532656] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:46.532673] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:46.532679] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:46.533289] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:46.533301] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:46.533307] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:46.539055] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224145251796000, replayable_scn:1755224145251796000, readable_scn:1755224145251796000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224146535580, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224145251796000, replayable_scn:1755224145251796000, readable_scn:1755224145251796000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224146535580, ora_rowscn_=1755224146431386000, content_changed=true) [2025-08-15 10:15:46.539230] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:46.539242] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:46.539247] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:46.544811] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:46.544824] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:46.544830] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:46.550120] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:46.550134] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:46.550140] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:46.622573] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:711) [728059][T1017_BackupDat][T1017][YB420ACB0122-000633A355859845-0-0] [lt=13] [DATA_BACKUP]success get jobs(job_attrs=[]) [2025-08-15 10:15:46.690161] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6876-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.701483] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD88-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.706931] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:46.706954] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:46.706961] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:46.750531] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF061-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.750594] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD97-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.776816] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C6-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:46.778258] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709533][T1014_TimerWK0_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:46.779367] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C6-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:46.795587] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A8-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.797189] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:46.798117] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A8-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:46.798135] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A8-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.814674] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8874-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.817214] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8874-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:46.817230] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8874-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.828376] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=6] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:46.829606] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224146828301, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:15:46.829628] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224151829627, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:15:46.848292] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D17-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.848537] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3B-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.851103] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3B-0-0] [lt=69] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:46.851121] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:46.851924] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8376-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.854977] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8376-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.856897] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8376-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.857044] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8376-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:46.900050] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E8-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:46.900161] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD525D-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:46.902534] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E8-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:46.902663] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD525D-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:46.915005] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B371-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:46.973381] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8693-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:46.976917] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52FC-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:46.977042] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8693-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:46.979430] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52FC-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:46.991640] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58E2-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:46.992396] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9C5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.065350] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA2-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:47.067980] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA2-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:47.104088] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58DC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.104786] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A6-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:47.107330] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:47.131790] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=12] mview_mds_op(tx_count=0) [2025-08-15 10:15:47.131803] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=8] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=821) [2025-08-15 10:15:47.133023] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727132][T1016_TimerWK0_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:47.136421] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0BD-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.148817] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF05E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.185559] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D15-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.200211] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9CA-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.203972] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:47.258338] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:47.318248] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487605][T1004_TimerWK2_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=22] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:47.330997] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A15-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.331796] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6D-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.333217] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:47.333527] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A15-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:47.333543] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A15-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.334336] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6D-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:47.334351] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6D-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.334426] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224147333146, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:15:47.334448] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224152334446, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:15:47.419524] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:47.420761] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224147419453, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:15:47.420788] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224152420786, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:15:47.426753] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E60-0-0] [lt=16] get number(ret=0, service_type_=0, range=1, base_id=1755224147426620000, start_id=1755224147426620000, end_id=1755224147426620001) [2025-08-15 10:15:47.427438] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E60-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224147426620000, v:0}, tenant_id=1017) [2025-08-15 10:15:47.435402] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7856-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.435456] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04F-0-0] [lt=9] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224147337394001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224147337394001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224147337394001, v:0}}], last_dump_ts_=1755224142403973, force_dump=false) [2025-08-15 10:15:47.435473] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.435483] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04F-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.435828] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E8-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.438458] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7856-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224147337394001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224147337394001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224147337394001, v:0}}], last_dump_ts_=1755224142407058, force_dump=false) [2025-08-15 10:15:47.438479] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7856-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.438507] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1D-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.438587] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.438689] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E8-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.439239] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C7-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.440546] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7856-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.440575] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.440595] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.440634] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7856-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.440661] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E8-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.440676] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC04F-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.440742] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E8-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.440812] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1D-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.441384] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6874-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.442291] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C7-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.444243] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C7-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.444331] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C7-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.460575] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224145334885005, replayable_scn:1755224145334885005, readable_scn:1755224145334885005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224147457117, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224145334885005, replayable_scn:1755224145334885005, readable_scn:1755224145334885005, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224147457117, ora_rowscn_=1755224146853979000, content_changed=true) [2025-08-15 10:15:47.460765] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:47.460776] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:47.460781] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:47.466036] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E400-0-0] [lt=13] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}) [2025-08-15 10:15:47.466075] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E400-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:15:47.493538] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7646-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.501164] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0BB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.501184] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E72A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.511089] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7646-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.513233] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7646-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.513339] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7646-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.523470] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8850-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.526190] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8850-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:47.526207] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8850-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.527683] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2F-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.530182] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2F-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:47.530198] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A2F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.532519] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224146335837002, replayable_scn:1755224146335837002, readable_scn:1755224146335837002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224147529691, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224146335837002, replayable_scn:1755224146335837002, readable_scn:1755224146335837002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224147529691, ora_rowscn_=1755224147509671000, content_changed=true) [2025-08-15 10:15:47.532684] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:47.532695] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:47.532702] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:47.533157] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224146335837001, replayable_scn:1755224146335837001, readable_scn:1755224146335837001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224147529691, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224146335837001, replayable_scn:1755224146335837001, readable_scn:1755224146335837001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224147529691, ora_rowscn_=1755224147441334000, content_changed=true) [2025-08-15 10:15:47.533325] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:47.533335] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:47.533340] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:47.534635] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:47.539224] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:47.539235] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:47.539244] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:47.544711] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224146335837001, replayable_scn:1755224146335837001, readable_scn:1755224146335837001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224147541226, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224146335837001, replayable_scn:1755224146335837001, readable_scn:1755224146335837001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224147541226, ora_rowscn_=1755224147437117000, content_changed=true) [2025-08-15 10:15:47.544879] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:47.544889] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:47.544897] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:47.546172] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:47.550066] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224146335837001, replayable_scn:1755224146335837001, readable_scn:1755224146335837001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224147546464, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224146335837001, replayable_scn:1755224146335837001, readable_scn:1755224146335837001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224147546464, ora_rowscn_=1755224147437117000, content_changed=true) [2025-08-15 10:15:47.550215] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:47.550224] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:47.550231] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:47.550239] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1006, cost_time_us=154, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:47.561460] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E69-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224147561185000, start_id=1755224147561185000, end_id=1755224147561185001) [2025-08-15 10:15:47.562128] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E69-0-0] [lt=13] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224147561185000, v:0}, tenant_id=1015) [2025-08-15 10:15:47.562540] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66B-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224147562240000, start_id=1755224147562240000, end_id=1755224147562240001) [2025-08-15 10:15:47.563194] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66B-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224147562240000, v:0}, tenant_id=1005) [2025-08-15 10:15:47.563217] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66B-0-0] [lt=13] zone_merge_mgr is already loaded(tenant_id=1005, global_merge_info={tenant_id:1005, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401690636, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194754861110, need_update:false}}, zone_merge_infos=[{tenant_id:1005, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194754863631, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401763272, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1005, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194754867140, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401768897, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1005, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194754868967, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401771210, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401612599000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:15:47.690239] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6877-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.701556] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD89-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.706753] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224146335837001, replayable_scn:1755224146335837001, readable_scn:1755224146335837001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224147704027, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224146335837001, replayable_scn:1755224146335837001, readable_scn:1755224146335837001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224147704027, ora_rowscn_=1755224147437117000, content_changed=true) [2025-08-15 10:15:47.706906] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:47.706917] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:47.706924] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:47.733838] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC63F-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.750607] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF062-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.750668] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD98-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.765335] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [728061][T1017_ArchiveSv][T1017][YB420ACB0122-000633A355A44E12-0-0] [lt=7] start do ObArchiveSchedulerService round(round=101447) [2025-08-15 10:15:47.768858] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [728061][T1017_ArchiveSv][T1017][YB420ACB0122-000633A355A44E12-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:15:47.768875] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [728061][T1017_ArchiveSv][T1017][YB420ACB0122-000633A355A44E12-0-0] [lt=13][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1018 AND dest_no = 0 AND name = 'state', exec_tenant_id=1017) [2025-08-15 10:15:47.768918] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [728061][T1017_ArchiveSv][T1017][YB420ACB0122-000633A355A44E12-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1018, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:47.768932] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [728061][T1017_ArchiveSv][T1017][YB420ACB0122-000633A355A44E12-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1018, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:15:47.768951] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [728061][T1017_ArchiveSv][T1017][YB420ACB0122-000633A355A44E12-0-0] [lt=15][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1018, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:47.784802] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C7-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:47.787371] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C7-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:47.790844] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36C9-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224147790261000, start_id=1755224147790261000, end_id=1755224147790261001) [2025-08-15 10:15:47.791496] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36C9-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224147790261000, v:0}, tenant_id=1011) [2025-08-15 10:15:47.797882] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:47.799192] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224147797802, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:15:47.799217] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224152799215, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:15:47.799271] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:47.800430] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224147799192, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:15:47.800451] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224152800450, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:15:47.800780] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A9-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.803288] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A9-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:47.803306] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95A9-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.819921] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8875-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.822427] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8875-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:47.822442] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8875-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.848362] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D18-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.853885] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3C-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.856394] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3C-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:47.856412] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:47.858096] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8377-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.861225] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8377-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.863183] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8377-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.863282] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8377-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:47.907914] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E9-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:47.907987] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD525E-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:47.908816] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB1-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224147908021000, start_id=1755224147908021000, end_id=1755224147908021001) [2025-08-15 10:15:47.909470] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB1-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224147908021000, v:0}, tenant_id=1013) [2025-08-15 10:15:47.910536] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD525E-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:47.911629] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8E9-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:47.916372] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B372-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:47.961272] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:47.962551] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224147961199, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:15:47.962581] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224152962579, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:15:47.982349] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8694-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:47.984766] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52FD-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:47.984881] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8694-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:47.987308] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52FD-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:47.991714] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58E3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:47.992471] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9C6-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.047080] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:48.048297] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224148047005, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:15:48.048321] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224153048320, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:15:48.073355] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA3-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:48.075895] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA3-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:48.087357] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B1-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224148086902000, start_id=1755224148086902000, end_id=1755224148086902001) [2025-08-15 10:15:48.088185] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B1-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224148086902000, v:0}, tenant_id=1012) [2025-08-15 10:15:48.104167] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58DD-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.107296] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463B9-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224148106874000, start_id=1755224148106874000, end_id=1755224148106874001) [2025-08-15 10:15:48.108089] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463B9-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224148106874000, v:0}, tenant_id=1) [2025-08-15 10:15:48.112610] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A7-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:48.115083] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A7-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:48.136498] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0BE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.148903] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF05F-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.185632] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D16-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.200284] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9CB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.243963] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8270-0-0] [lt=8] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1005, thread_idx=0) [2025-08-15 10:15:48.245236] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1005, only_for_display=false) [2025-08-15 10:15:48.246451] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:48.260218] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.261382] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.262539] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.262562] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1005, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:48.262580] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.262592] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1005, ls_id:{id:1}, replicas:[{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:48.262695] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=80] build ls locality stat map(tenant_id=1005, ls_id={id:1}) [2025-08-15 10:15:48.262704] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.262717] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.262728] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.262740] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:48.262817] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.262831] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.262841] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.262851] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.262860] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3797, tenant_id=1005, ls_id={id:1}) [2025-08-15 10:15:48.262871] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:48.263936] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18700, tenant_id=1005, acc_dr_task=0) [2025-08-15 10:15:48.265156] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1006, only_for_display=false) [2025-08-15 10:15:48.266350] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:48.277374] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B634D-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90429688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}) [2025-08-15 10:15:48.277416] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B634D-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:48.280020] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.281171] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.282323] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.282346] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:48.282363] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.282369] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=4] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1}, replicas:[{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:48.282473] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=81] build ls locality stat map(tenant_id=1006, ls_id={id:1}) [2025-08-15 10:15:48.282481] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.282496] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.282506] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.282518] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:48.282595] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.282609] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.282618] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.282624] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.282634] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3761, tenant_id=1006, ls_id={id:1}) [2025-08-15 10:15:48.282643] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:48.283728] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:48.304369] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.305508] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.306683] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.306707] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:48.306723] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.306735] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1001}, replicas:[{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:48.306860] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=98] build ls locality stat map(tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:15:48.306869] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.306882] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.306896] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=11] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.306913] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:48.307012] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=74] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.307029] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.307040] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.307050] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.307059] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3822, tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:15:48.307071] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:48.308133] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:48.329509] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.330673] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.331815] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.331840] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:48.331857] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.331869] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1002}, replicas:[{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:48.331979] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=86] build ls locality stat map(tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:15:48.331987] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.332001] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.332012] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.332024] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:48.332101] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.332115] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.332125] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.332130] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.332139] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3759, tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:15:48.332150] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:48.332918] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227748331700}) [2025-08-15 10:15:48.332934] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:48.332952] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:15:48.333218] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:48.336255] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A16-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.337010] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6E-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.338814] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A16-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:48.338832] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A16-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.338924] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227748337751}) [2025-08-15 10:15:48.338942] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:48.338952] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:15:48.339524] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6E-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:48.339546] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6E-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.340142] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227748338963}) [2025-08-15 10:15:48.340156] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:48.340165] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17348][TimerWK1_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:15:48.353999] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.355160] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.356333] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.356357] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:48.356373] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.356385] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1003}, replicas:[{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:48.356486] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=79] build ls locality stat map(tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:15:48.356494] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.356507] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.356518] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.356529] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:48.356607] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.356621] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.356630] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.356635] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.356644] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3783, tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:15:48.356655] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:48.357726] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93768, tenant_id=1006, acc_dr_task=0) [2025-08-15 10:15:48.441472] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6875-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.441638] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7857-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.441646] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC050-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.441860] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.441858] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E9-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224148360408000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224148360408000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224148360408000, v:0}}], last_dump_ts_=1755224143410413, force_dump=false) [2025-08-15 10:15:48.441881] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E9-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.444720] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7857-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.444789] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC050-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.444822] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E9-0-0] [lt=7] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224148360408000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224148360408000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224148360408000, v:0}}], last_dump_ts_=1755224143413265, force_dump=false) [2025-08-15 10:15:48.444843] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E9-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.444865] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1E-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.445194] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C8-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.446755] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E9-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.446764] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7857-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.446855] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76E9-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.446860] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7857-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.446890] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC050-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.446935] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.446987] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC050-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.447087] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1E-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.448205] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C8-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.450051] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C8-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.450139] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C8-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.460815] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:48.460828] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:48.460836] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:48.501236] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0BC-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.501257] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E72B-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.507906] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF219-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.514284] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7647-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.517313] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7647-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.519212] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7647-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.519309] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7647-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.526550] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A8A-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81054688GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81445312GB}) [2025-08-15 10:15:48.526589] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A8A-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:15:48.528384] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8851-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.531232] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8851-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:48.531255] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8851-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.532776] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:48.532789] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:48.532795] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:48.533206] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A30-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.533455] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:48.533469] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:48.533479] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:48.535727] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A30-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:48.535746] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A30-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.539324] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:48.539337] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:48.539342] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:48.540614] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:48.544871] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:48.544884] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:48.544890] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:48.550258] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:48.550276] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:48.550286] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:48.605057] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:48.606341] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224148604982, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:15:48.606366] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224153606364, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:15:48.612215] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DEF-0-0] [lt=11] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1, thread_idx=0) [2025-08-15 10:15:48.613466] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=17] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-08-15 10:15:48.614665] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:48.628985] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.630133] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.631280] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:48.631302] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:48.631316] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.631328] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:48.631430] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=79] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-08-15 10:15:48.631437] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.631450] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.631458] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:48.631469] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:48.631547] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.631566] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=10] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.631576] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.631586] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:48.631593] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3799, tenant_id=1, ls_id={id:1}) [2025-08-15 10:15:48.631603] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:48.632759] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19293, tenant_id=1, acc_dr_task=0) [2025-08-15 10:15:48.690317] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6878-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.701632] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD8A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.707034] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:48.707052] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:48.707064] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:48.744139] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E59-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224148743389000, start_id=1755224148743389000, end_id=1755224148743389001) [2025-08-15 10:15:48.744763] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E59-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224148743389000, v:0}, tenant_id=1016) [2025-08-15 10:15:48.750683] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF063-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.750740] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD99-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.792735] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C8-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:48.795321] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C8-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:48.806003] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AA-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.808566] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AA-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:48.808584] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AA-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.825162] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8876-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.827687] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8876-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:48.827703] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8876-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.848434] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D19-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.859157] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3D-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.861647] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3D-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:48.861666] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3D-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:48.864254] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8378-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.867377] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8378-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224148339337005, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224148840260000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224148840260000, v:0}}], last_dump_ts_=1755224143836270, force_dump=false) [2025-08-15 10:15:48.867403] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8378-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.869486] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8378-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.869599] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8378-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:48.915900] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD525F-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:48.916951] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8EA-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:48.917647] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=11] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224148536129, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224147337394004, replayable_scn:1755224147337394004, readable_scn:1755224147337394004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:48.917746] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B373-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:48.918392] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD525F-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:48.919436] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8EA-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:48.990154] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8695-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:48.991787] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58E4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.992555] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9C7-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:48.992614] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52FE-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:48.992647] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8695-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:48.995127] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52FE-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:49.000150] INFO [RS] process (ob_root_inspection.cpp:410) [17454][RSInspectTask][T0][Y0-000634393DD76AF6-0-0] [lt=5] purge recyclebin task begin [2025-08-15 10:15:49.000168] INFO [RS] process (ob_root_inspection.cpp:428) [17454][RSInspectTask][T0][Y0-000634393DD76AF6-0-0] [lt=11] submit purge recyclebin task success(delay=60000000) [2025-08-15 10:15:49.000173] INFO [RS] process (ob_root_inspection.cpp:430) [17454][RSInspectTask][T0][Y0-000634393DD76AF6-0-0] [lt=4] purge recyclebin task end(delay=60000000) [2025-08-15 10:15:49.081291] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA4-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:49.083888] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA4-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:49.104243] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58DE-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.120380] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A8-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:49.122869] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A8-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:49.136568] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0BF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.148977] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF060-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.158628] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64C-0-0] [lt=16] get number(ret=0, service_type_=0, range=1, base_id=1755224149158000000, start_id=1755224149158000000, end_id=1755224149158000001) [2025-08-15 10:15:49.159406] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64C-0-0] [lt=14] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224149158000000, v:0}, tenant_id=1006) [2025-08-15 10:15:49.185706] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D17-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.200357] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9CC-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.214140] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E62-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1003, thread_idx=0) [2025-08-15 10:15:49.215433] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1003, only_for_display=false) [2025-08-15 10:15:49.216675] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:49.223302] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A5-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224149223211000, start_id=1755224149223211000, end_id=1755224149223211001) [2025-08-15 10:15:49.224113] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A5-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224149223211000, v:0}, tenant_id=1008) [2025-08-15 10:15:49.230487] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.231647] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.232810] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.232831] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1003, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:49.232845] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.232854] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1003, ls_id:{id:1}, replicas:[{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:49.232963] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=85] build ls locality stat map(tenant_id=1003, ls_id={id:1}) [2025-08-15 10:15:49.232971] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.232982] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.232990] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.232999] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:49.233075] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.233087] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.233095] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.233100] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.233107] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3784, tenant_id=1003, ls_id={id:1}) [2025-08-15 10:15:49.233114] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:49.234163] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18731, tenant_id=1003, acc_dr_task=0) [2025-08-15 10:15:49.235384] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1004, only_for_display=false) [2025-08-15 10:15:49.236599] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:49.250295] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.251516] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.252670] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.252691] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:49.252706] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.252715] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:49.252821] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=82] build ls locality stat map(tenant_id=1004, ls_id={id:1}) [2025-08-15 10:15:49.252831] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.252842] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.252850] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.252859] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:49.252933] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=54] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.252952] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=11] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.252959] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.252965] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.252974] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3839, tenant_id=1004, ls_id={id:1}) [2025-08-15 10:15:49.252985] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:49.254055] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:49.274980] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.276150] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.277320] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.277343] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:49.277358] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.277366] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:49.277469] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=80] build ls locality stat map(tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:15:49.277477] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.277490] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.277498] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.277508] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:49.277587] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.277600] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.277607] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.277614] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.277618] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3804, tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:15:49.277626] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:49.278675] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:49.299575] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.300775] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.301922] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.301950] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:49.301967] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.301976] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1002}, replicas:[{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:49.302085] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=85] build ls locality stat map(tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:15:49.302093] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.302104] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.302112] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.302121] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:49.302198] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.302210] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.302217] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.302224] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.302230] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3805, tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:15:49.302238] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:49.303324] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=3] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:49.324198] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.325415] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.326600] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.326623] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:49.326639] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.326648] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1003}, replicas:[{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:49.326755] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=83] build ls locality stat map(tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:15:49.326763] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.326774] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.326782] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.326791] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:49.326866] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.326877] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.326884] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.326891] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.326898] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3863, tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:15:49.326905] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:49.327984] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93807, tenant_id=1004, acc_dr_task=0) [2025-08-15 10:15:49.341642] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A17-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.342264] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6F-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.344210] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A17-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:49.344227] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A17-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.344779] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6F-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:49.344794] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF6F-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.419765] INFO [RS] finish (ob_mview_maintenance_task.cpp:280) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=14] mview maintenace task finish(tenant_id_=1004, round_=116350, status_=100, error_code_=0, cost_us_=20001388, prepare_cost_us_=1, gc_mview_cost_us_=1224, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:15:49.441551] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6876-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.447929] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EA-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.447923] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7858-0-0] [lt=5] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224149340995000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224149340995000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224149340995000, v:0}}], last_dump_ts_=1755224144416349, force_dump=false) [2025-08-15 10:15:49.447954] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7858-0-0] [lt=23] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.447972] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC051-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.448069] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1F-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224149255731000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224149340995000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224149340995000, v:0}}], last_dump_ts_=1755224144416891, force_dump=false) [2025-08-15 10:15:49.448098] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1F-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.451123] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.451234] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7858-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.451309] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EA-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.451319] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC051-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.451385] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1F-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224149340995000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224149340995001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224149340995001, v:0}}], last_dump_ts_=1755224144419895, force_dump=false) [2025-08-15 10:15:49.451409] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1F-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.453274] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EA-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.453375] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EA-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.453472] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1F-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.453582] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C1F-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.453892] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC051-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.453910] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7858-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.453969] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC051-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.454018] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7858-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.454065] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C9-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.455961] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C9-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.456232] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413C9-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.460870] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:49.460882] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:49.460890] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:49.462203] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:49.466510] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E411-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}) [2025-08-15 10:15:49.466549] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E411-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2025-08-15 10:15:49.501311] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0BD-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.501332] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E72C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.520351] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7648-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224149317819000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224149317819000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224149317819000, v:0}}], last_dump_ts_=1755224144475241, force_dump=false) [2025-08-15 10:15:49.520378] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7648-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.523481] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7648-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224149317819000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224149317819000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224149317819000, v:0}}], last_dump_ts_=1755224144478265, force_dump=false) [2025-08-15 10:15:49.523504] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7648-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.525458] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7648-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.525573] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7648-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.532777] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:49.532791] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:49.532797] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:49.533462] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:49.533478] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:49.533483] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:49.533461] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8852-0-0] [lt=47] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.534101] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:49.536104] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8852-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:49.536129] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8852-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.538469] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A31-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.539178] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224148339337003, replayable_scn:1755224148339337003, readable_scn:1755224148339337003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224149535734, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224148339337003, replayable_scn:1755224148339337003, readable_scn:1755224148339337003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224149535734, ora_rowscn_=1755224149449337000, content_changed=true) [2025-08-15 10:15:49.539348] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:49.539363] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:49.539371] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:49.540989] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A31-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:49.541008] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A31-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.544981] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:49.544994] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:49.545000] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:49.546974] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BB-0-0] [lt=13] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1017, thread_idx=0) [2025-08-15 10:15:49.548248] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1017, only_for_display=false) [2025-08-15 10:15:49.549503] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:49.550262] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:49.550274] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:49.550282] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:49.551639] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:49.563598] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.564795] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.565992] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.566019] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1017, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:49.566037] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.566048] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1017, ls_id:{id:1}, replicas:[{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:49.566165] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=92] build ls locality stat map(tenant_id=1017, ls_id={id:1}) [2025-08-15 10:15:49.566176] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.566187] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.566195] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.566204] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:49.566280] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.566292] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.566299] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.566304] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.566311] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3908, tenant_id=1017, ls_id={id:1}) [2025-08-15 10:15:49.566319] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:49.567418] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19170, tenant_id=1017, acc_dr_task=0) [2025-08-15 10:15:49.568620] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1018, only_for_display=false) [2025-08-15 10:15:49.569804] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:49.583832] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.584999] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.586196] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.586219] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:49.586235] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.586246] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1}, replicas:[{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:49.586351] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=82] build ls locality stat map(tenant_id=1018, ls_id={id:1}) [2025-08-15 10:15:49.586362] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.586373] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.586382] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.586391] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:49.586466] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.586478] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.586486] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.586491] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.586495] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3813, tenant_id=1018, ls_id={id:1}) [2025-08-15 10:15:49.586502] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:49.587576] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:49.608926] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.610087] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=23] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.611228] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.611250] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:49.611270] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=13] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.611282] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1001}, replicas:[{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:49.611385] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=79] build ls locality stat map(tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:15:49.611393] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.611406] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.611417] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.611429] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:49.611505] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.611519] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.611529] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.611536] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.611544] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3772, tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:15:49.611550] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:49.612624] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:49.625780] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:49.627006] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224149625706, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:15:49.627031] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224154627029, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:15:49.633870] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.635084] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.636274] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.636296] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:49.636310] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.636318] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1002}, replicas:[{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:49.636426] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=84] build ls locality stat map(tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:15:49.636437] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.636448] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.636456] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.636464] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:49.636544] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=59] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.636557] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.636564] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.636569] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.636576] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3882, tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:15:49.636586] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:49.637647] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:49.658790] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=8] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.659989] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.661162] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:49.661187] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:49.661205] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.661217] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1003}, replicas:[{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:49.661322] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=81] build ls locality stat map(tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:15:49.661330] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.661344] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.661356] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:49.661363] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:49.661440] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.661454] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.661464] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.661469] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:49.661477] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3879, tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:15:49.661483] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:49.662620] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=95190, tenant_id=1018, acc_dr_task=0) [2025-08-15 10:15:49.690397] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6879-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.701712] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD8B-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.707055] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:49.707068] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:49.707077] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:49.708519] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:49.718519] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E5-0-0] [lt=17] get number(ret=0, service_type_=0, range=1, base_id=1755224149717710000, start_id=1755224149717710000, end_id=1755224149717710001) [2025-08-15 10:15:49.719243] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E5-0-0] [lt=19] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224149717710000, v:0}, tenant_id=1003) [2025-08-15 10:15:49.738733] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC640-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.750760] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF064-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.750813] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD9A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.781743] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C399A-0-0] [lt=12] [RESTORE] restore scheduler quit [2025-08-15 10:15:49.783019] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C399A-0-0] [lt=14] get targets(ret=0, jobs=[], exec_tenant_id=1003, sql=select * from __all_recover_table_job) [2025-08-15 10:15:49.784211] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C399B-0-0] [lt=14] get targets(ret=0, jobs=[], exec_tenant_id=1003, sql=select * from __all_import_table_job) [2025-08-15 10:15:49.784235] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C399C-0-0] [lt=14][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224149784230, ref_timestamp=1755224089779694, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:15:49.784318] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C399D-0-0] [lt=12] [RESTORE] restore scheduler start [2025-08-15 10:15:49.784329] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C399E-0-0] [lt=8] [RESTORE] try process restore job [2025-08-15 10:15:49.785469] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [1487703][T1003_REST_SER][T1003][YB420ACB0122-0006337FE32C399E-0-0] [lt=4] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:15:49.800512] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C9-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:49.803060] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38C9-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:49.811322] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AB-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.813866] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AB-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:49.813883] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AB-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.830435] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8877-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.833511] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8877-0-0] [lt=97] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:49.833530] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8877-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.848509] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D1A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.858945] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C2-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224149858638000, start_id=1755224149858638000, end_id=1755224149858638001) [2025-08-15 10:15:49.859732] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C2-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224149858638000, v:0}, tenant_id=1004) [2025-08-15 10:15:49.864409] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3E-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.866947] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3E-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:49.866965] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3E-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:49.870570] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8379-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224149340995004, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224149841804000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224149841804000, v:0}}], last_dump_ts_=1755224144839306, force_dump=false) [2025-08-15 10:15:49.870597] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8379-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.873603] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8379-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.875545] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8379-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.875648] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8379-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:49.919177] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B374-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:49.923737] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5260-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:49.924616] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8EB-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:49.926228] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5260-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:49.927101] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8EB-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:49.991858] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58E5-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.992628] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9C8-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:49.998006] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8696-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:50.000466] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52FF-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:50.000539] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8696-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:50.002390] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17738][T1_TimerWK1_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=6] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:50.002972] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD52FF-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:50.089280] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA5-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:50.091796] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA5-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:50.104318] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58DF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.128215] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A9-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:50.130712] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62A9-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:50.136642] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0C0-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.149051] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF061-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.185782] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D18-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.190268] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C3-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224150190207000, start_id=1755224150190207000, end_id=1755224150190207001) [2025-08-15 10:15:50.190948] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476154][T1007_FrzInfoDe][T1007][YB420ACB0122-00063382124FA2C3-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224150190207000, v:0}, tenant_id=1007) [2025-08-15 10:15:50.200429] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9CD-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.277859] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D51-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.90820312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}) [2025-08-15 10:15:50.277900] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-0006342767FB9D51-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2025-08-15 10:15:50.346933] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A18-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.347479] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF70-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.349510] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A18-0-0] [lt=74] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:50.349529] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A18-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.350002] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF70-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:50.350019] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF70-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.441626] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6877-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.454395] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EB-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.454518] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C20-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.454954] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC052-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.455019] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7859-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.457339] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CA-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.457581] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C20-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.457686] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EB-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.457923] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7859-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.457996] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC052-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.459651] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C20-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.459655] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EB-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.459743] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C20-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.459748] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EB-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.459848] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7859-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.459926] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7859-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.459958] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC052-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.460084] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC052-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.460645] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CA-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.460810] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224148839207001, replayable_scn:1755224148839207001, readable_scn:1755224148839207001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224150457304, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224148839207001, replayable_scn:1755224148839207001, readable_scn:1755224148839207001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224150457304, ora_rowscn_=1755224149872322000, content_changed=true) [2025-08-15 10:15:50.461006] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:50.461017] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:50.461022] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:50.462734] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CA-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.462833] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CA-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.501387] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0BE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.501408] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E72D-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.508219] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF21A-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.526549] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7649-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.527030] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A8C-0-0] [lt=11] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81445312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81835938GB}) [2025-08-15 10:15:50.527070] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A8C-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:15:50.529522] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7649-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.531644] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7649-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.531730] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7649-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.532772] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224149317819000, replayable_scn:1755224149317819000, readable_scn:1755224149317819000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224150529650, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224149317819000, replayable_scn:1755224149317819000, readable_scn:1755224149317819000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224150529650, ora_rowscn_=1755224150528595000, content_changed=true) [2025-08-15 10:15:50.532951] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:50.532965] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:50.532971] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:50.533412] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224149340995000, replayable_scn:1755224149340995000, readable_scn:1755224149340995000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224150530707, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224149340995000, replayable_scn:1755224149340995000, readable_scn:1755224149340995000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224150530707, ora_rowscn_=1755224150459414000, content_changed=true) [2025-08-15 10:15:50.533578] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:50.533589] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:50.533597] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:50.538876] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8853-0-0] [lt=48] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.539412] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:50.539422] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:50.539430] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:50.541583] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8853-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:50.541603] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8853-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.543810] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A32-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.544863] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224149339941002, replayable_scn:1755224149339941002, readable_scn:1755224149339941002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224150541240, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224149339941002, replayable_scn:1755224149339941002, readable_scn:1755224149339941002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224150541240, ora_rowscn_=1755224150456250000, content_changed=true) [2025-08-15 10:15:50.545046] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:50.545056] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:50.545061] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:50.546370] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A32-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:50.546383] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A32-0-0] [lt=10] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.550190] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224149340995001, replayable_scn:1755224149340995001, readable_scn:1755224149340995001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224150547559, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224149340995001, replayable_scn:1755224149340995001, readable_scn:1755224149340995001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224150547559, ora_rowscn_=1755224150456250000, content_changed=true) [2025-08-15 10:15:50.550359] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:50.550369] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:50.550380] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:50.550392] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1006, cost_time_us=178, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:50.620749] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:50.621997] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224150620674, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:15:50.622023] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224155622021, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:15:50.690475] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C687A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.695886] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [3414015][T1006_TimerWK4_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:50.701788] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD8C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.706975] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224149340995000, replayable_scn:1755224149340995000, readable_scn:1755224149340995000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224150703362, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224149340995000, replayable_scn:1755224149340995000, readable_scn:1755224149340995000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224150703362, ora_rowscn_=1755224150456250000, content_changed=true) [2025-08-15 10:15:50.707168] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:50.707181] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:50.707187] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:50.750835] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF065-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.750883] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD9B-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.808489] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38CA-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:50.812172] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38CA-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:50.816574] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AC-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.819153] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AC-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:50.819172] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AC-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.836210] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8878-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.838704] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8878-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:50.838721] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8878-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.848588] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D1B-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.869724] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3F-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.872230] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3F-0-0] [lt=66] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:50.872247] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB3F-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:50.876644] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.879659] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.881718] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.881822] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:50.920587] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B375-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:50.931526] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5261-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:50.932349] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8EC-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:50.934026] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5261-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:50.934840] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8EC-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:50.945391] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:50.946614] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224150945320, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:15:50.946639] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224155946637, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:15:50.991927] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58E6-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:50.992704] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9C9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.005828] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8697-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:51.006618] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:51.007861] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224150996914, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:15:51.007881] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224156007879, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:15:51.007896] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224156007893, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:51.008314] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8697-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:51.008329] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5300-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:51.010827] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5300-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:51.097186] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA6-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:51.099667] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA6-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:51.104395] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58E0-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.136235] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62AA-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:51.136703] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0C1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.138731] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62AA-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:51.149125] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF062-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.185851] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D19-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.188233] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:51.189468] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224151188160, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:15:51.189492] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224156189490, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:15:51.190754] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:51.191948] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=6] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224151190685, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:15:51.191967] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224156191966, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:51.200461] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9CE-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.344485] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227751343296}) [2025-08-15 10:15:51.344502] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:51.344511] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:15:51.350464] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227751349285}) [2025-08-15 10:15:51.350480] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:51.350488] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:15:51.351667] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227751350497}) [2025-08-15 10:15:51.351681] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:51.351693] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:15:51.352269] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A19-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.352692] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF71-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.354795] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A19-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:51.354813] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A19-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.355194] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF71-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:51.355213] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF71-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.402401] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E50-0-0] [lt=10] get number(ret=0, service_type_=0, range=1, base_id=1755224151402315000, start_id=1755224151402315000, end_id=1755224151402315001) [2025-08-15 10:15:51.403139] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728184][T1018_FrzInfoDe][T1018][YB420ACB0122-000633A3563A7E50-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224151402315000, v:0}, tenant_id=1018) [2025-08-15 10:15:51.441697] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6878-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.460780] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C21-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.460785] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EC-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.460957] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.460967] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:51.460983] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:51.460990] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:51.461027] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC053-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.463794] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CB-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224151343355001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224151343355001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224151343355001, v:0}}], last_dump_ts_=1755224146433298, force_dump=false) [2025-08-15 10:15:51.463822] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CB-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.463848] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C21-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.463915] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EC-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.464026] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC053-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224151343355001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224151343355002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224151343355002, v:0}}], last_dump_ts_=1755224146432346, force_dump=false) [2025-08-15 10:15:51.464048] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC053-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.464110] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.465847] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C21-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.465961] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C21-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.465965] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EC-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.466077] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.466097] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC053-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.466141] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EC-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.466163] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.466206] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC053-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.466742] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CB-0-0] [lt=17] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224151343355001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224151343355001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224151343355001, v:0}}], last_dump_ts_=1755224146436303, force_dump=false) [2025-08-15 10:15:51.466765] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CB-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.466997] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E413-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}) [2025-08-15 10:15:51.467035] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437F2D8E413-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=40, ddl_arg=NULL) [2025-08-15 10:15:51.468622] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CB-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.468733] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CB-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.496711] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D2-0-0] [lt=18] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1015, thread_idx=0) [2025-08-15 10:15:51.498003] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1015, only_for_display=false) [2025-08-15 10:15:51.499232] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:51.501460] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0BF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.501481] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E72E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.513125] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.514275] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.515441] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.515465] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1015, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:51.515482] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.515493] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1015, ls_id:{id:1}, replicas:[{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1015, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:51.515605] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=87] build ls locality stat map(tenant_id=1015, ls_id={id:1}) [2025-08-15 10:15:51.515614] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.515627] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.515639] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.515650] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151410846456, create_time_us:1750151405385054, tenant_id:1015, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151405411946, create_time_us:1750151405385573, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151406110772, create_time_us:1750151405386766, tenant_id:1015, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:51.515728] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.515742] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.515752] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.515757] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.515766] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3793, tenant_id=1015, ls_id={id:1}) [2025-08-15 10:15:51.515775] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:51.516843] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18840, tenant_id=1015, acc_dr_task=0) [2025-08-15 10:15:51.518110] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1016, only_for_display=false) [2025-08-15 10:15:51.519326] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:51.532692] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764A-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.533093] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.533626] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:51.533647] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:51.533658] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:51.534253] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.535442] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.535465] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:51.535483] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.535494] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1}, replicas:[{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:51.535619] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=97] build ls locality stat map(tenant_id=1016, ls_id={id:1}) [2025-08-15 10:15:51.535630] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.535643] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.535655] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.535672] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764A-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.535666] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151411638632, create_time_us:1750151411638632, tenant_id:1016, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411639009, create_time_us:1750151411639009, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411683661, create_time_us:1750151411683661, tenant_id:1016, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:51.535752] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=63] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.535766] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.535775] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.535780] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.535788] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3837, tenant_id=1016, ls_id={id:1}) [2025-08-15 10:15:51.535799] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:51.536896] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:51.537560] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764A-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.537649] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.539012] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:51.539030] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:51.539036] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:51.544398] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8854-0-0] [lt=46] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.545044] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:51.545056] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:51.545061] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:51.547009] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8854-0-0] [lt=68] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:51.547029] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8854-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.549086] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A33-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.550462] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:51.550475] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:51.550480] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:51.551598] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A33-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:51.551615] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A33-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.557977] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.559131] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.560277] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.560300] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:51.560316] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.560327] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1001}, replicas:[{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:51.560429] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=80] build ls locality stat map(tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:15:51.560437] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.560450] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.560456] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.560467] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151417263533, create_time_us:1750151411780616, tenant_id:1016, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151411983085, create_time_us:1750151411785112, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411786132, tenant_id:1016, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:51.560544] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.560558] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.560568] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.560573] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.560582] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3780, tenant_id=1016, ls_id={id:1001}) [2025-08-15 10:15:51.560595] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=10] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:51.561669] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=9] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:51.583792] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.584951] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.586096] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.586119] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:51.586135] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.586147] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1002}, replicas:[{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:51.586248] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=79] build ls locality stat map(tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:15:51.586258] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.586271] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.586282] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.586289] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=4] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151420931034, create_time_us:1750151411920827, tenant_id:1016, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421381650, create_time_us:1750151411954614, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412016492, create_time_us:1750151411915500, tenant_id:1016, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:51.586368] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.586382] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.586391] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.586395] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.586404] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3782, tenant_id=1016, ls_id={id:1002}) [2025-08-15 10:15:51.586410] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:51.587660] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:51.608458] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.609604] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.610793] INFO [RS] get (ob_server_table_operator.cpp:820) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:51.610816] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1016, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:51.610828] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.610839] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1016, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1016, ls_id:{id:1003}, replicas:[{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:51.610953] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=89] build ls locality stat map(tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:15:51.610962] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.610975] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.610982] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:51.610993] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151421021737, create_time_us:1750151412008394, tenant_id:1016, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1024, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1024, in_pool:true, unit:unit_id:1024, resource_pool_id:1024, unit_group_id:1008, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151412985476, create_time_us:1750151412013655, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1023, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1023, in_pool:true, unit:unit_id:1023, resource_pool_id:1023, unit_group_id:1008, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151421483428, create_time_us:1750151412003858, tenant_id:1016, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1022, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1022, in_pool:true, unit:unit_id:1022, resource_pool_id:1022, unit_group_id:1008, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:51.611075] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=59] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.611089] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.611099] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.611104] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:51.611112] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3785, tenant_id=1016, ls_id={id:1003}) [2025-08-15 10:15:51.611120] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:51.612191] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [727238][T1015_DRService][T1015][YB420ACB0122-000633A351FA68D3-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=95333, tenant_id=1016, acc_dr_task=0) [2025-08-15 10:15:51.690551] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C687B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.701869] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD8D-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.707121] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:51.707134] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:51.707143] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:51.743616] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC641-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.750878] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF066-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.750955] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD9C-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.779602] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:51.796535] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA0-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224151796477000, start_id=1755224151796477000, end_id=1755224151796477001) [2025-08-15 10:15:51.797365] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709761][T1014_FrzInfoDe][T1014][YB420ACB0122-000633A33F1A7FA0-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224151796477000, v:0}, tenant_id=1014) [2025-08-15 10:15:51.802624] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:51.817956] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38CB-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:51.820709] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38CB-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:51.821833] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AD-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.824339] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AD-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:51.824358] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AD-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.829703] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:51.830908] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224151829627, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:15:51.830931] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224156830930, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:15:51.841350] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8879-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.843839] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8879-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:51.843854] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8879-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.848662] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D1C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.874968] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB40-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.877456] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB40-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:51.877473] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB40-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:51.882783] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.885791] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.887777] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837B-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.887972] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837B-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:51.919129] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=11] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224151535850, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224150253298000, replayable_scn:1755224150253298000, readable_scn:1755224150253298000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:51.922027] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B376-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:51.939334] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5262-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:51.940045] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8ED-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:51.941808] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5262-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:51.942536] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8ED-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:51.992010] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58E7-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:51.992775] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9CA-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.013568] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8698-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:52.016064] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8698-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:52.016117] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5301-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:52.018701] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5301-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:52.104438] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58E1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.105044] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA7-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:52.107544] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA7-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:52.136767] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0C2-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.137629] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727133][T1016_TimerWK1_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=15] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:52.143971] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62AB-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:52.146523] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62AB-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:52.149200] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF063-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.185928] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D1A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.200512] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9CF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.205302] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=16] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:52.259681] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619487][T1012_TimerWK2_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:52.278314] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA13-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91210938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}) [2025-08-15 10:15:52.278357] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA13-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:52.319569] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487604][T1004_TimerWK1_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=5] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:52.334524] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:52.335769] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224152334446, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:15:52.335792] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224157335790, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:15:52.340934] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC5-0-0] [lt=7] start do ObArchiveSchedulerService round(round=195066) [2025-08-15 10:15:52.343323] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC5-0-0] [lt=21][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:15:52.343342] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC5-0-0] [lt=14][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1 AND dest_no = 0 AND name = 'state', exec_tenant_id=1) [2025-08-15 10:15:52.343392] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC5-0-0] [lt=11][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:52.343407] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC5-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:15:52.343418] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [17714][T1_ArchiveSvr][T1][YB420ACB0122-0006337F0415BBC5-0-0] [lt=9][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:52.357499] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1A-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.357870] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF72-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.360000] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1A-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:52.360018] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.360361] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF72-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:52.360379] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF72-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.420863] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:52.422091] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224152420786, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:15:52.422115] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224157422114, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:15:52.441772] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6879-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.460992] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:52.461007] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:52.461012] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:52.466982] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C22-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.467149] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76ED-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.467168] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785B-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.467166] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC054-0-0] [lt=5] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224152344583001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224152344583002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224152344583002, v:0}}], last_dump_ts_=1755224147435455, force_dump=false) [2025-08-15 10:15:52.467242] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC054-0-0] [lt=56] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.469757] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CC-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.470149] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C22-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.470191] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76ED-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.470266] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC054-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.470329] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785B-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224152344583001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224151843780005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224152344583001, v:0}}], last_dump_ts_=1755224147438457, force_dump=false) [2025-08-15 10:15:52.470356] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785B-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.472152] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76ED-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.472208] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC054-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.472243] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76ED-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.472303] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC054-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.472404] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785B-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.472496] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.472676] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C22-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.472765] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C22-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.473386] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CC-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.475251] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CC-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.475340] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CC-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.501538] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0C0-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.501557] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E72F-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.508493] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF21B-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.527538] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A8E-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.81835938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82226562GB}) [2025-08-15 10:15:52.527576] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E8DCD6A8E-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:15:52.533462] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:52.533474] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:52.533479] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:52.533617] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:52.533629] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:52.533636] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:52.534923] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:52.538630] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764B-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.539009] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:52.539025] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:52.539030] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:52.541620] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764B-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.543604] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764B-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.543687] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.545102] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:52.545116] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:52.545121] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:52.546420] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:52.549697] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8855-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.550468] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:52.550481] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:52.550487] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:52.552341] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8855-0-0] [lt=71] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:52.552359] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8855-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.554936] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A34-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.558174] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A34-0-0] [lt=92] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:52.558192] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A34-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.588023] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69E9-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1013, thread_idx=0) [2025-08-15 10:15:52.589291] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1013, only_for_display=false) [2025-08-15 10:15:52.590518] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:52.604350] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.605532] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.606704] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.606727] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1013, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:52.606746] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.606758] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1013, ls_id:{id:1}, replicas:[{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1013, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:52.606863] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=82] build ls locality stat map(tenant_id=1013, ls_id={id:1}) [2025-08-15 10:15:52.606872] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.606887] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.606898] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.606913] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148118703691, create_time_us:1750148113263611, tenant_id:1013, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113921772, create_time_us:1750148113263404, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148113301338, create_time_us:1750148113257140, tenant_id:1013, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:52.607011] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=73] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.607026] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.607047] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=16] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.607057] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.607062] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3871, tenant_id=1013, ls_id={id:1}) [2025-08-15 10:15:52.607073] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:52.608163] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18872, tenant_id=1013, acc_dr_task=0) [2025-08-15 10:15:52.609378] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1014, only_for_display=false) [2025-08-15 10:15:52.610594] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:52.624371] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.625541] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.626719] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.626739] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:52.626754] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.626764] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1}, replicas:[{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:52.626874] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=85] build ls locality stat map(tenant_id=1014, ls_id={id:1}) [2025-08-15 10:15:52.626881] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.626890] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.626898] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.626907] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148120092475, create_time_us:1750148120092475, tenant_id:1014, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120094478, create_time_us:1750148120094478, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148120090445, create_time_us:1750148120090445, tenant_id:1014, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:52.626988] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=60] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.627000] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.627008] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.627015] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.627020] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3800, tenant_id=1014, ls_id={id:1}) [2025-08-15 10:15:52.627029] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:52.628119] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:52.648947] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.650112] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.651270] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.651293] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:52.651309] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.651320] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1001}, replicas:[{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:52.651443] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=96] build ls locality stat map(tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:15:52.651452] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.651465] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.651475] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.651487] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148141574539, create_time_us:1750148136085550, tenant_id:1014, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137461451, create_time_us:1750148136067136, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148137429953, create_time_us:1750148136080800, tenant_id:1014, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:52.651567] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=59] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.651582] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.651591] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.651597] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.651607] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3824, tenant_id=1014, ls_id={id:1001}) [2025-08-15 10:15:52.651618] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:52.652698] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:52.673874] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.675045] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.676194] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.676217] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:52.676234] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.676245] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1002}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:52.676348] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=80] build ls locality stat map(tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:15:52.676356] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.676369] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.676380] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.676389] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137218315, tenant_id:1014, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142673937, create_time_us:1750148137197110, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138432679, create_time_us:1750148137197503, tenant_id:1014, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:52.676465] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.676477] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.676485] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.676492] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.676499] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3760, tenant_id=1014, ls_id={id:1002}) [2025-08-15 10:15:52.676506] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:52.677586] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:52.690629] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C687C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.698340] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.699492] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.700658] INFO [RS] get (ob_server_table_operator.cpp:820) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:52.700681] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1014, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:52.700697] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.700709] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1014, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1014, ls_id:{id:1003}, replicas:[{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:52.700812] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=80] build ls locality stat map(tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:15:52.700820] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.700833] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.700842] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:52.700856] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750148137873832, create_time_us:1750148137574468, tenant_id:1014, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1019, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1019, in_pool:true, unit:unit_id:1019, resource_pool_id:1019, unit_group_id:1007, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148138463835, create_time_us:1750148137499392, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1021, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1021, in_pool:true, unit:unit_id:1021, resource_pool_id:1021, unit_group_id:1007, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750148142991489, create_time_us:1750148137495239, tenant_id:1014, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1020, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1020, in_pool:true, unit:unit_id:1020, resource_pool_id:1020, unit_group_id:1007, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:52.700932] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.700952] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=11] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.700961] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.700967] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:52.700975] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3778, tenant_id=1014, ls_id={id:1003}) [2025-08-15 10:15:52.700986] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:52.701947] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD8E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.702051] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [709636][T1013_DRService][T1013][YB420ACB0122-000633A341CA69EA-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93875, tenant_id=1014, acc_dr_task=0) [2025-08-15 10:15:52.707200] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:52.707212] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:52.707221] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:52.751029] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD9D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.751054] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF067-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.799297] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:52.800553] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224152799215, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:15:52.800576] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224157800574, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:15:52.801747] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224152800450, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:15:52.801767] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=15] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224157801766, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:15:52.826005] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38CC-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:52.827063] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AE-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.828502] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38CC-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:52.829583] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AE-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:52.829599] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AE-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.846604] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887A-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.848732] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D1D-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.849159] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887A-0-0] [lt=74] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:52.849176] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887A-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.880247] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB41-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.882769] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB41-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:52.882785] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB41-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:52.888945] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.891508] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:340) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837C-0-0] [lt=14] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1014, ls_id:{id:1}, sync_scn:{val:1755224152845152001, v:0}, readable_scn:{val:1755224152845152001, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-08-15 10:15:52.891896] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.893817] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.893922] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:52.893966] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:459) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837C-0-0] [lt=6] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1755224150842142001, v:0}) [2025-08-15 10:15:52.923439] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B377-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:52.947848] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8EE-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:52.948877] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5263-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:52.950350] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8EE-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:52.954347] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5263-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:52.962652] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:52.963967] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224152962579, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:15:52.963991] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224157963989, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:15:52.992081] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58E8-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:52.992850] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9CB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.022627] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8699-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:53.026449] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB8699-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:53.026509] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5302-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:53.029000] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5302-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:53.048395] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:53.049652] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224153048320, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:15:53.049677] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224158049675, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:15:53.104513] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58E2-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.112985] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA8-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:53.116661] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA8-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:53.136837] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0C3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.149271] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF064-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.151847] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62AC-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:53.154367] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62AC-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:53.186011] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D1B-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.200563] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9D0-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.362759] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1B-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.363060] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF73-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.365278] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1B-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:53.365296] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1B-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.365610] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF73-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:53.365630] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF73-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.441848] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C687A-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.460898] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224151843780001, replayable_scn:1755224151843780001, readable_scn:1755224151843780001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224153457768, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224151843780001, replayable_scn:1755224151843780001, readable_scn:1755224151843780001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224153457768, ora_rowscn_=1755224152890428000, content_changed=true) [2025-08-15 10:15:53.461095] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:53.461109] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:53.461117] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:53.467463] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98CFA-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.93164062GB}) [2025-08-15 10:15:53.467500] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-0006343789A98CFA-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:15:53.473267] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EE-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224153346272001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224153346272001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224153346272001, v:0}}], last_dump_ts_=1755224148441857, force_dump=false) [2025-08-15 10:15:53.473296] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EE-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.473327] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC055-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.473480] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.473677] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C23-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.476304] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CD-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.476453] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.476539] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EE-0-0] [lt=7] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224153346272001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224153346272002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224153346272002, v:0}}], last_dump_ts_=1755224148444821, force_dump=false) [2025-08-15 10:15:53.476557] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC055-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.476565] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EE-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.476627] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C23-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.478443] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785C-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.478539] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785C-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.478562] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EE-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.478601] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C23-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.478613] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC055-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.478643] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EE-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.478756] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC055-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.478813] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C23-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.479325] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CD-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.481280] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CD-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.481362] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CD-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.501615] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0C1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.501631] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E730-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.531848] INFO [RS] finish (ob_mview_maintenance_task.cpp:280) [2619486][T1012_TimerWK1_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=41] mview maintenace task finish(tenant_id_=1012, round_=77507, status_=100, error_code_=0, cost_us_=20001361, prepare_cost_us_=0, gc_mview_cost_us_=1232, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:15:53.532856] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224151343355001, replayable_scn:1755224151343355001, readable_scn:1755224151343355001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224153530256, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224151343355001, replayable_scn:1755224151343355001, readable_scn:1755224151343355001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224153530256, ora_rowscn_=1755224152540221000, content_changed=true) [2025-08-15 10:15:53.533041] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=19] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:53.533053] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:53.533060] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:53.533529] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224152344583001, replayable_scn:1755224152344583001, readable_scn:1755224152344583001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224153530256, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224152344583001, replayable_scn:1755224152344583001, readable_scn:1755224152344583001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224153530256, ora_rowscn_=1755224153477708000, content_changed=true) [2025-08-15 10:15:53.533699] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:53.533711] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:53.533716] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:53.538892] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224152344583001, replayable_scn:1755224152344583001, readable_scn:1755224152344583001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224153536576, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224152344583001, replayable_scn:1755224152344583001, readable_scn:1755224152344583001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224153536576, ora_rowscn_=1755224153475601000, content_changed=true) [2025-08-15 10:15:53.539062] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:53.539073] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:53.539079] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:53.544701] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764C-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.545026] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224152344583001, replayable_scn:1755224152344583001, readable_scn:1755224152344583001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224153541887, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224152344583001, replayable_scn:1755224152344583001, readable_scn:1755224152344583001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224153541887, ora_rowscn_=1755224153475601000, content_changed=true) [2025-08-15 10:15:53.545199] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:53.545210] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:53.545215] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:53.547753] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764C-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.549819] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764C-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.549915] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764C-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.550395] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224152344583002, replayable_scn:1755224152344583002, readable_scn:1755224152344583002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224153547113, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224152344583002, replayable_scn:1755224152344583002, readable_scn:1755224152344583002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224153547113, ora_rowscn_=1755224153475601000, content_changed=true) [2025-08-15 10:15:53.550556] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:53.550568] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:53.550573] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:53.550584] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1006, cost_time_us=165, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:53.555080] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8856-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.557589] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8856-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:53.557607] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8856-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.560827] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A35-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.563331] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A35-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:53.563351] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A35-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.606438] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:53.607689] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224153606364, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:15:53.607712] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224158607711, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:15:53.690666] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C687D-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.700963] INFO [RS] gc_mview (ob_mview_maintenance_task.cpp:267) [709534][T1014_TimerWK1_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=10] mview maintenance task gc mview success(tenant_id_=1014, round_=63450, gc_mview_cost_us_=1220, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2025-08-15 10:15:53.702024] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD8F-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.707137] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224152344583001, replayable_scn:1755224152344583001, readable_scn:1755224152344583001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224153703749, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224152344583001, replayable_scn:1755224152344583001, readable_scn:1755224152344583001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224153703749, ora_rowscn_=1755224153475601000, content_changed=true) [2025-08-15 10:15:53.707322] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:53.707337] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:53.707344] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:53.748552] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC642-0-0] [lt=17] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.751102] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD9E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.751130] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF068-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.832269] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AF-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.833875] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38CD-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:53.834768] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AF-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:53.834786] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95AF-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.836391] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38CD-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:53.848807] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D1E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.851898] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887B-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.854512] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887B-0-0] [lt=68] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:53.854528] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887B-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.885534] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB42-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.888048] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB42-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:53.888064] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB42-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:53.894897] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837D-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.897870] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837D-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224153846661001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224153846661001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224153846661001, v:0}}], last_dump_ts_=1755224148867376, force_dump=false) [2025-08-15 10:15:53.897896] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837D-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.899792] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837D-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.899887] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:53.924846] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B378-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:53.955785] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8EF-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:53.958387] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8EF-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:53.959617] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5264-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:53.962092] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5264-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:53.992159] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58E9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:53.992922] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9CC-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.033140] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB869A-0-0] [lt=15] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:54.036719] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5303-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:54.036796] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB869A-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:54.040386] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5303-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:54.104591] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58E3-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.122056] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA9-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:54.124557] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FA9-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:54.136909] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0C4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.149343] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF065-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.159730] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62AD-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:54.163366] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62AD-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:54.186076] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D1C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.200637] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9D1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.278795] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B635C-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91601562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}) [2025-08-15 10:15:54.278835] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-00063427983B635C-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=44, ddl_arg=NULL) [2025-08-15 10:15:54.356063] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=15][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227754354873}) [2025-08-15 10:15:54.356081] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:54.356090] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:15:54.362087] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227754360905}) [2025-08-15 10:15:54.362102] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:54.362113] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:15:54.363440] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=16][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227754362124}) [2025-08-15 10:15:54.363461] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=16][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:54.363477] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:15:54.368054] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1C-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.368328] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF74-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.370600] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1C-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:54.370617] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1C-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.370878] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF74-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:54.370896] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF74-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.441922] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C687B-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.461098] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:54.461116] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:54.461123] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:54.462580] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:54.479514] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785D-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224154347313001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224154347313001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224154347313001, v:0}}], last_dump_ts_=1755224149447922, force_dump=false) [2025-08-15 10:15:54.479535] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785D-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.479595] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EF-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.479730] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC056-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.479754] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C24-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224154258940000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224154258940000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224154258940000, v:0}}], last_dump_ts_=1755224149448068, force_dump=false) [2025-08-15 10:15:54.479782] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C24-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.482390] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CE-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.482594] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.482819] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EF-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.482810] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C24-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224154258940000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224154258940000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224154258940000, v:0}}], last_dump_ts_=1755224149451384, force_dump=false) [2025-08-15 10:15:54.482837] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C24-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.482990] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC056-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.484620] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.484701] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785D-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.484906] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C24-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.484917] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EF-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.485029] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C24-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.485037] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC056-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.485064] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76EF-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.485150] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC056-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.485358] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CE-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.487332] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CE-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.487421] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CE-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.501693] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0C2-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.501701] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E731-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.508762] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF21C-0-0] [lt=10] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.528107] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1293-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82226562GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}) [2025-08-15 10:15:54.528145] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341ED71D1293-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:15:54.533124] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:54.533137] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:54.533143] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:54.533743] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:54.533757] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:54.533766] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:54.534420] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:54.539096] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:54.539109] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:54.539114] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:54.540385] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:54.545208] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:54.545221] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:54.545227] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:54.550550] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:54.550563] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:54.550569] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:54.550932] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764D-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224154469422001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224154469422001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224154469422001, v:0}}], last_dump_ts_=1755224149520350, force_dump=false) [2025-08-15 10:15:54.550963] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764D-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.551832] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:54.553857] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764D-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224154469422000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224154469422000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224154469422000, v:0}}], last_dump_ts_=1755224149523481, force_dump=false) [2025-08-15 10:15:54.553880] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764D-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.556055] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764D-0-0] [lt=4] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.556149] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764D-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.560316] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8857-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.562813] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8857-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:54.562832] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8857-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.566024] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A36-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.568548] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A36-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:54.568567] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A36-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.580277] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:104) [17458][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=13] start checking lost replicas [2025-08-15 10:15:54.581531] INFO [RS.LB] idle (ob_root_balancer.cpp:87) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=36] root balance idle(idle_time=10000000) [2025-08-15 10:15:54.581548] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:70) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=12] start check unit migrate finish [2025-08-15 10:15:54.581558] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=6] try check migrate unit finish by tenant(tenant_id=1) [2025-08-15 10:15:54.590850] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1004) [2025-08-15 10:15:54.616631] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1006) [2025-08-15 10:15:54.627106] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:54.628327] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224154627029, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:15:54.628351] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224159628349, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:15:54.642477] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=14] try check migrate unit finish by tenant(tenant_id=1008) [2025-08-15 10:15:54.668313] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=15] try check migrate unit finish by tenant(tenant_id=1012) [2025-08-15 10:15:54.690738] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C687E-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.694276] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=16] try check migrate unit finish by tenant(tenant_id=1014) [2025-08-15 10:15:54.702103] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD90-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.707383] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:54.707395] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:54.707403] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:54.708698] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:54.720000] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1016) [2025-08-15 10:15:54.745616] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:163) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=13] try check migrate unit finish by tenant(tenant_id=1018) [2025-08-15 10:15:54.751180] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFD9F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.751203] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF069-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.771308] INFO [RS] check (ob_alter_locality_finish_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81518-0-0] [lt=16] start to check alter locality finish [2025-08-15 10:15:54.772469] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81519-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.773629] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81519-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.775917] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8151B-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.777071] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8151B-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.779386] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8151D-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.780541] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8151D-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.782824] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8151F-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.783990] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8151F-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.786305] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81521-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.787453] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81521-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.789727] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81523-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.790869] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81523-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.793144] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81525-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.794283] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81525-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.796541] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81527-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.797681] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81527-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.798865] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:71) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81527-0-0] [lt=13] start check shrink resource pool [2025-08-15 10:15:54.798882] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81528-0-0] [lt=11] start check shrink resource pool(tenant_id=1) [2025-08-15 10:15:54.800004] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81528-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.801197] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81528-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.801261] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81529-0-0] [lt=15] start check shrink resource pool(tenant_id=1004) [2025-08-15 10:15:54.802386] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81529-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.803540] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81529-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.803592] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152A-0-0] [lt=12] start check shrink resource pool(tenant_id=1006) [2025-08-15 10:15:54.804702] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152A-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.805859] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152A-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.805911] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152B-0-0] [lt=12] start check shrink resource pool(tenant_id=1008) [2025-08-15 10:15:54.807032] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152B-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.808180] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152B-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.808233] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152C-0-0] [lt=12] start check shrink resource pool(tenant_id=1012) [2025-08-15 10:15:54.809344] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152C-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.810675] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152C-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.810729] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152D-0-0] [lt=12] start check shrink resource pool(tenant_id=1014) [2025-08-15 10:15:54.811859] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152D-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.813054] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152D-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.813107] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152E-0-0] [lt=11] start check shrink resource pool(tenant_id=1016) [2025-08-15 10:15:54.814230] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152E-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.815410] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152E-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.815470] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:92) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152F-0-0] [lt=11] start check shrink resource pool(tenant_id=1018) [2025-08-15 10:15:54.816595] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152F-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.817765] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152F-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.817817] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:99) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152F-0-0] [lt=11] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.817828] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [17459][RootBalance][T0][YB420ACB0122-00063437B1A8152F-0-0] [lt=8] start check primary zone [2025-08-15 10:15:54.817838] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81530-0-0] [lt=6] start check primary zone(tenant_id=1) [2025-08-15 10:15:54.818974] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81530-0-0] [lt=6][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.819024] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81531-0-0] [lt=10] start check primary zone(tenant_id=1004) [2025-08-15 10:15:54.820127] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81531-0-0] [lt=6][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.820186] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81532-0-0] [lt=13] start check primary zone(tenant_id=1006) [2025-08-15 10:15:54.821291] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81532-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.821341] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81533-0-0] [lt=10] start check primary zone(tenant_id=1008) [2025-08-15 10:15:54.822445] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81533-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.822494] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81534-0-0] [lt=11] start check primary zone(tenant_id=1012) [2025-08-15 10:15:54.823607] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81534-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.823656] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81535-0-0] [lt=10] start check primary zone(tenant_id=1014) [2025-08-15 10:15:54.824753] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81535-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.824802] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81536-0-0] [lt=11] start check primary zone(tenant_id=1016) [2025-08-15 10:15:54.825966] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81536-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.826016] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=11] start check primary zone(tenant_id=1018) [2025-08-15 10:15:54.827120] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:337) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2025-08-15 10:15:54.827171] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:166) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=11] start do unit balance [2025-08-15 10:15:54.833393] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:350) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=7] start distribute for server status change [2025-08-15 10:15:54.833493] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:401) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=13] finish distribute for server status change(ret=0) [2025-08-15 10:15:54.833504] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:200) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=8] finish do unit balance(ret=0) [2025-08-15 10:15:54.837538] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B0-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.839554] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:115) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=7] start to do tenant group unit balance [2025-08-15 10:15:54.839598] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=13] start to do tenant group unit balance(zone=zone1) [2025-08-15 10:15:54.839612] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=8] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.203.1.34:2882", disk_in_use_:3147825152, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:54.839630] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=12] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:54.839660] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:54.839678] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=9] generate available servers finish(ret=0, zone=zone1, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:54.839688] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone1) [2025-08-15 10:15:54.839699] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=7] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:54.839719] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=12] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.93164062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.1.34:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.1.34:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.93164062GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:54.839760] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=33] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.203.1.34:2882", disk_in_use_:3147825152, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:54.839772] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=10] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.1.34:2882", disk_in_use_:3147825152, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:54.839782] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=5] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:54.839795] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=10] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1006, unit_config:{unit_config_id:1005, name:"config_opsone_zone1_s_2_4_ywx", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7988, unit_config:0x7f1e9aa71c38, pool:0x7f1c3beb2f18}, {unit_id:1008, tenant_id:1006, resource_pool_id:1008, unit_config:{unit_config_id:1008, name:"config_test_zone1_s_2_4_emp", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f77d8, unit_config:0x7f1e9aa71998, pool:0x7f1c3beb27a8}, {unit_id:1011, tenant_id:1008, resource_pool_id:1011, unit_config:{unit_config_id:1011, name:"config_mcp_zone1_s_4_8_tmc", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7550, unit_config:0x7f1e9aa716f8, pool:0x7f1e443fda40}, {unit_id:1016, tenant_id:1012, resource_pool_id:1016, unit_config:{unit_config_id:1017, name:"config_oms_zone1_s_4_8_gzp", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f73a0, unit_config:0x7f1e9aa71458, pool:0x7f1e443fd2d0}, {unit_id:1019, tenant_id:1014, resource_pool_id:1019, unit_config:{unit_config_id:1019, name:"config_grafana_test_zone1_s_1_2_pri", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7118, unit_config:0x7f1e9aa71298, pool:0x7f1e443fc7a8}, {unit_id:1024, tenant_id:1016, resource_pool_id:1024, unit_config:{unit_config_id:1023, name:"config_temporalio_zone1_s_2_4_ykw", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6ce0, unit_config:0x7f1e9aa70f18, pool:0x7f1e74dff2d0}, {unit_id:1026, tenant_id:1018, resource_pool_id:1026, unit_config:{unit_config_id:1027, name:"config_media_zone1_s_2_4_kmn", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6b30, unit_config:0x7f1e9aa70b98, pool:0x7f1e74dfeb60}], not_grant_units=[], available_servers=["10.203.1.34:2882"]) [2025-08-15 10:15:54.839853] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=34] start to do tenant group unit balance(zone=zone2) [2025-08-15 10:15:54.839862] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=6] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.203.4.38:2882", disk_in_use_:3135242240, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:54.839874] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=8] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:54.839891] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=9] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:54.839903] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=6] generate available servers finish(ret=0, zone=zone2, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:54.839913] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=7] start do non-tenantgroup unit balance task(zone=zone2) [2025-08-15 10:15:54.839917] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=4] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:54.839930] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.38:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.38:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:54.839968] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=30] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.203.4.38:2882", disk_in_use_:3135242240, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:54.839974] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=4] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.38:2882", disk_in_use_:3135242240, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:54.839984] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:54.839995] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=8] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1004, unit_config:{unit_config_id:1004, name:"config_opsone_zone2_s_2_4_wag", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7b38, unit_config:0x7f1e9aa71d18, pool:0x7f1c3beb3688}, {unit_id:1009, tenant_id:1006, resource_pool_id:1009, unit_config:{unit_config_id:1009, name:"config_test_zone2_s_2_4_yey", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7700, unit_config:0x7f1e9aa718b8, pool:0x7f1c3beb23f0}, {unit_id:1010, tenant_id:1008, resource_pool_id:1010, unit_config:{unit_config_id:1010, name:"config_mcp_zone2_s_4_8_bjb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7628, unit_config:0x7f1e9aa717d8, pool:0x7f1c3beb2038}, {unit_id:1018, tenant_id:1012, resource_pool_id:1018, unit_config:{unit_config_id:1016, name:"config_oms_zone2_s_4_8_kbx", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f71f0, unit_config:0x7f1e9aa71538, pool:0x7f1e443fcb60}, {unit_id:1021, tenant_id:1014, resource_pool_id:1021, unit_config:{unit_config_id:1021, name:"config_grafana_test_zone2_s_1_2_yyc", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f6f68, unit_config:0x7f1e9aa710d8, pool:0x7f1e443fc038}, {unit_id:1023, tenant_id:1016, resource_pool_id:1023, unit_config:{unit_config_id:1022, name:"config_temporalio_zone2_s_2_4_lsv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6db8, unit_config:0x7f1e9aa70ff8, pool:0x7f1e74dff688}, {unit_id:1027, tenant_id:1018, resource_pool_id:1027, unit_config:{unit_config_id:1025, name:"config_media_zone2_s_2_4_yzd", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6a58, unit_config:0x7f1e9aa70d58, pool:0x7f1e74dfe7a8}], not_grant_units=[], available_servers=["10.203.4.38:2882"]) [2025-08-15 10:15:54.840046] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:918) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=30] start to do tenant group unit balance(zone=zone3) [2025-08-15 10:15:54.840055] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7923) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=7] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.203.4.39:2882", disk_in_use_:3034578944, disk_total_:1439585796096, wild_server_:false}], over_disk_waterlevel_:false}) [2025-08-15 10:15:54.840065] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1072) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=7] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004, 1005, 1006, 1007, 1008, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018], group_tenant_array=[]) [2025-08-15 10:15:54.840081] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=8] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:54.840094] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1256) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=7] generate available servers finish(ret=0, zone=zone3, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:54.840103] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5299) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=6] start do non-tenantgroup unit balance task(zone=zone3) [2025-08-15 10:15:54.840107] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5628) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=3] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=5.739896051826824941e-01, mem_weights=3.714583444482716934e-01, log_disk_weights=5.455205036904582638e-02, data_disk_weights=0.000000000000000000e+00, available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:54.840120] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4535) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=9] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}], available_servers=["10.203.4.39:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.203.4.39:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:21, max_cpu:21, memory_size:"39GB", log_disk_size:"111.491733GB", data_disk_size:0, min_iops:-8, max_iops:-8, iops_weight:20, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:21, }}}, resource_info_:{cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}, inter_ttg_load_value_:"3.585593094031713779e-01"}]) [2025-08-15 10:15:54.840150] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8114) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=25] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.203.4.39:2882", disk_in_use_:3034578944, disk_total_:1439585796096, wild_server_:false}, over_disk_waterlevel=false) [2025-08-15 10:15:54.840156] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8120) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=4] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.203.4.39:2882", disk_in_use_:3034578944, disk_total_:1439585796096, wild_server_:false}]) [2025-08-15 10:15:54.840166] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5281) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=7, not_grant_units.count()=0) [2025-08-15 10:15:54.840176] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5336) [17459][RootBalance][T0][YB420ACB0122-00063437B1A81537-0-0] [lt=8] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=3.764872748733299468e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1005, unit_config:{unit_config_id:1006, name:"config_opsone_zone3_s_2_4_vxv", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f7a60, unit_config:0x7f1e9aa71b58, pool:0x7f1c3beb32d0}, {unit_id:1007, tenant_id:1006, resource_pool_id:1007, unit_config:{unit_config_id:1007, name:"config_test_zone3_s_2_4_mui", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f78b0, unit_config:0x7f1e9aa71a78, pool:0x7f1c3beb2b60}, {unit_id:1012, tenant_id:1008, resource_pool_id:1012, unit_config:{unit_config_id:1012, name:"config_mcp_zone3_s_4_8_seu", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f7478, unit_config:0x7f1e9aa71618, pool:0x7f1e443fd688}, {unit_id:1017, tenant_id:1012, resource_pool_id:1017, unit_config:{unit_config_id:1018, name:"config_oms_zone3_s_4_8_nvb", resource:{min_cpu:4, max_cpu:4, memory_size:"8GB", log_disk_size:"24GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:4, }}, unit:0x7f1a2f9f72c8, unit_config:0x7f1e9aa71378, pool:0x7f1e443fcf18}, {unit_id:1020, tenant_id:1014, resource_pool_id:1020, unit_config:{unit_config_id:1020, name:"config_grafana_test_zone3_s_1_2_xbz", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0x7f1a2f9f7040, unit_config:0x7f1e9aa711b8, pool:0x7f1e443fc3f0}, {unit_id:1022, tenant_id:1016, resource_pool_id:1022, unit_config:{unit_config_id:1024, name:"config_temporalio_zone3_s_2_4_rxz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6e90, unit_config:0x7f1e9aa70e38, pool:0x7f1e74dffa40}, {unit_id:1025, tenant_id:1018, resource_pool_id:1025, unit_config:{unit_config_id:1026, name:"config_media_zone3_s_2_4_ukz", resource:{min_cpu:2, max_cpu:2, memory_size:"4GB", log_disk_size:"12GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:2, }}, unit:0x7f1a2f9f6c08, unit_config:0x7f1e9aa70c78, pool:0x7f1e74dfef18}], not_grant_units=[], available_servers=["10.203.4.39:2882"]) [2025-08-15 10:15:54.840703] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B0-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:54.840721] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B0-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.841746] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38CE-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:54.844254] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38CE-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:54.848879] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D1F-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.857214] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887C-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.859751] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887C-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:54.859769] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887C-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.882806] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=19] start do ObArchiveSchedulerService round(round=168692) [2025-08-15 10:15:54.884102] INFO [ARCHIVE] checkpoint_ (ob_tenant_archive_scheduler.cpp:682) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=13] schedule archive(round_info={key:{tenant_id:1008, dest_no:0}, incarnation:1, dest_id:1001, round_id:1, state:{status:"DOING"}, start_scn:{val:1752696006021581000, v:0}, checkpoint_scn:{val:1755223995192401003, v:0}, max_scn:{val:1755224059715728003, v:0}, compatible:{version:1}, base_piece_id:1, used_piece_id:30, piece_switch_interval:86400000000, frozen_input_bytes:13323961132, frozen_output_bytes:13323961132, active_input_bytes:117351157, active_output_bytes:117351157, deleted_input_bytes:0, deleted_output_bytes:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com", comment:""}) [2025-08-15 10:15:54.885550] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476034][T1007_ArchiveSv][T1008][YB420ACB0122-0006338212A554BF-0-0] [lt=21] get number(ret=0, service_type_=0, range=1, base_id=1755224154884896000, start_id=1755224154884896000, end_id=1755224154884896001) [2025-08-15 10:15:54.885570] INFO [SHARE] get_backup_scn (ob_backup_struct.cpp:2155) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=16] get tenant gts(ret=0, ret="OB_SUCCESS", tenant_id=1008, scn={val:1755224154884896000, v:0}) [2025-08-15 10:15:54.885589] INFO [SHARE] count_ (ob_archive_checkpoint.cpp:173) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=10] print count result(ret=0, counter={ls_count:4, deleted_ls_count:0, not_start_cnt:0, interrupted_cnt:0, doing_cnt:4, stopped_cnt:0, interrupted_ls_id:{id:-1}, max_scn:{val:1755224115293438002, v:0}, checkpoint_scn:{val:1755224040686907000, v:0}, max_active_piece_id:30}) [2025-08-15 10:15:54.886791] INFO [SHARE] get_backup_dest (ob_backup_connectivity.cpp:1268) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=12] success get backup dest(sql=SELECT authorization,extension FROM __all_backup_storage_info WHERE tenant_id = 1008 AND path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog' AND endpoint = 'host=cos.ap-beijing.myqcloud.com', tenant_id=1008, backup_dest={root_path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog", storage_info:{endpoint:"host=cos.ap-beijing.myqcloud.com", access_id:"access_id=AKIDiKl3A5ZXJ959EMuL3hJ1g9PTM5lFvjhT", extension:"appid=1254236265&checksum_type=md5", type:"COS", checksum_type:1, max_iops:0, max_bandwidth:0, role_arn:0x7f175cd18f90, external_id:0x7f175cd19090}}) [2025-08-15 10:15:54.890804] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB43-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.893303] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB43-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:54.893321] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB43-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:54.900909] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837E-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224154848160001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224154848160001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224154848160001, v:0}}], last_dump_ts_=1755224149870570, force_dump=false) [2025-08-15 10:15:54.900934] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837E-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.903867] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837E-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.905883] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.905964] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:54.916982] INFO mk_parent_dir (ob_backup_io_adapter.cpp:346) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=17] found parent dir(i=130, path="cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog/piece_d1001r1p30/checkpoint") [2025-08-15 10:15:54.920759] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224154536707, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224153346272002, replayable_scn:1755224153346272002, readable_scn:1755224153346272002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:54.926267] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B379-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:54.963665] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F0-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:54.966236] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F0-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:54.967367] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5265-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:54.969836] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5265-0-0] [lt=9] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:54.977865] INFO [STORAGE] print_access_storage_log (ob_storage.cpp:46) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=12] access storage op=write_single_file uri=cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog/piece_d1001r1p30/checkpoint/checkpoint_info.1755224040686907000.obarc size=0 Byte cost_ts=60857 us speed=0.00 MB/s [2025-08-15 10:15:54.977880] INFO [SHARE] write_checkpoint_file_ (ob_archive_checkpoint_mgr.cpp:255) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=11] succeed to write checkpoint file.(path={cur_pos:172, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog/piece_d1001r1p30/checkpoint/checkpoint_info.1755224040686907000.obarc"}) [2025-08-15 10:15:54.992232] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58EA-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:54.992999] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9CD-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.000221] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C57-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1011, thread_idx=0) [2025-08-15 10:15:55.001519] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1011, only_for_display=false) [2025-08-15 10:15:55.002765] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:55.003681] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [17737][T1_TimerWK0_TntSharedTimer][T1][Y0-0000000000000000-0-0] [lt=9] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:55.018605] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.019786] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.020950] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.020974] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1011, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:55.020991] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.021003] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1011, ls_id:{id:1}, replicas:[{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1011, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:55.021113] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=85] build ls locality stat map(tenant_id=1011, ls_id={id:1}) [2025-08-15 10:15:55.021121] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.021135] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.021145] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.021152] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525077587, create_time_us:1749023519616966, tenant_id:1011, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519635913, create_time_us:1749023519620116, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023519637751, create_time_us:1749023519621993, tenant_id:1011, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:55.021230] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.021245] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.021254] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.021260] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.021268] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3835, tenant_id=1011, ls_id={id:1}) [2025-08-15 10:15:55.021278] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:55.022729] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=21211, tenant_id=1011, acc_dr_task=0) [2025-08-15 10:15:55.023964] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1012, only_for_display=false) [2025-08-15 10:15:55.025171] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:55.040347] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.041500] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.042228] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB869B-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:55.042643] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.042665] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:55.042683] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.042695] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1}, replicas:[{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:55.042803] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=84] build ls locality stat map(tenant_id=1012, ls_id={id:1}) [2025-08-15 10:15:55.042812] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.042825] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.042838] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=10] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.042846] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023525848257, create_time_us:1749023525848257, tenant_id:1012, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525871292, create_time_us:1749023525871292, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023525859996, create_time_us:1749023525859996, tenant_id:1012, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:55.042925] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.042945] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.042955] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.042961] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.042969] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3781, tenant_id=1012, ls_id={id:1}) [2025-08-15 10:15:55.042976] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:55.044553] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:55.044775] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB869B-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:55.045714] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5304-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:55.048236] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5304-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:55.066769] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.067937] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.069105] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.069130] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:55.069147] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.069158] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1001}, replicas:[{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:55.069267] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=85] build ls locality stat map(tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:15:55.069276] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.069289] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.069295] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.069302] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023531462345, create_time_us:1749023526000041, tenant_id:1012, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526123690, create_time_us:1749023526007846, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023525999451, tenant_id:1012, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:55.069385] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=59] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.069399] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.069408] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.069415] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.069423] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3790, tenant_id=1012, ls_id={id:1001}) [2025-08-15 10:15:55.069433] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:55.070982] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:55.093732] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.094880] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.096030] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.096052] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:55.096070] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.096083] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1002}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:55.096194] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=86] build ls locality stat map(tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:15:55.096202] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.096215] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.096225] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.096235] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526064447, tenant_id:1012, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531563570, create_time_us:1749023526067967, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023526122436, create_time_us:1749023526062562, tenant_id:1012, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:55.096314] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.096328] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.096338] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.096344] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.096353] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3764, tenant_id=1012, ls_id={id:1002}) [2025-08-15 10:15:55.096363] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:55.097943] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:55.104669] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58E4-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.120400] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.121568] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.122714] INFO [RS] get (ob_server_table_operator.cpp:820) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.122741] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1012, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:55.122759] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.122771] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1012, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1012, ls_id:{id:1003}, replicas:[{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:55.122883] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=87] build ls locality stat map(tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:15:55.122892] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.122906] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.122916] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.122924] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1749023526877889, create_time_us:1749023526117198, tenant_id:1012, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1016, in_pool:true, unit:unit_id:1016, resource_pool_id:1016, unit_group_id:1006, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023527126465, create_time_us:1749023526118424, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1018, in_pool:true, unit:unit_id:1018, resource_pool_id:1018, unit_group_id:1006, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1749023531615529, create_time_us:1749023526119271, tenant_id:1012, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1017, in_pool:true, unit:unit_id:1017, resource_pool_id:1017, unit_group_id:1006, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:55.123012] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=64] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.123028] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.123037] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.123045] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.123054] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3843, tenant_id=1012, ls_id={id:1003}) [2025-08-15 10:15:55.123064] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:55.124648] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [2619612][T1011_DRService][T1011][YB420ACB0122-0006338CC41C1C58-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=101903, tenant_id=1012, acc_dr_task=0) [2025-08-15 10:15:55.129974] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FAA-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:55.132480] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FAA-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:55.136982] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0C5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.138716] WDIAG [STORAGE] print_access_storage_log_ (ob_i_storage.cpp:907) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=8][errcode=0] access object storage cost too much time: del_file (ob_storage.cpp:839), uri=cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog/piece_d1001r1p30/checkpoint/checkpoint_info.1755223995192401003.obarc, size=0 byte, start_time=1755224154977902, cost_ts=160812 us, speed=0.00 MB/s, is_slow=1 [2025-08-15 10:15:55.139990] INFO [SHARE] get_backup_dest (ob_backup_connectivity.cpp:1268) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=16] success get backup dest(sql=SELECT authorization,extension FROM __all_backup_storage_info WHERE tenant_id = 1008 AND path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog' AND endpoint = 'host=cos.ap-beijing.myqcloud.com', tenant_id=1008, backup_dest={root_path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog", storage_info:{endpoint:"host=cos.ap-beijing.myqcloud.com", access_id:"access_id=AKIDiKl3A5ZXJ959EMuL3hJ1g9PTM5lFvjhT", extension:"appid=1254236265&checksum_type=md5", type:"COS", checksum_type:1, max_iops:0, max_bandwidth:0, role_arn:0x7f1ab5ea8f90, external_id:0x7f1ab5ea9090}}) [2025-08-15 10:15:55.141384] INFO [SHARE] do_compare_and_swap_ (ob_inner_table_operator.cpp:937) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=20] compare and swap one column(key={key:{tenant_id:1008, dest_no:0}, incarnation:1, dest_id:1001, round_id:1, state:{status:"DOING"}, start_scn:{val:1752696006021581000, v:0}, checkpoint_scn:{val:1755223995192401003, v:0}, max_scn:{val:1755224059715728003, v:0}, compatible:{version:1}, base_piece_id:1, used_piece_id:30, piece_switch_interval:86400000000, frozen_input_bytes:13323961132, frozen_output_bytes:13323961132, active_input_bytes:117351157, active_output_bytes:117351157, deleted_input_bytes:0, deleted_output_bytes:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com", comment:""}, assignments="tenant_id = 1008, dest_no = 0, round_id = 1, incarnation = 1, dest_id = 1001, status = 'DOING', start_scn = 1752696006021581000, checkpoint_scn = 1755224040686907000, max_scn = 1755224115293438002, compatible = 1, base_piece_id = 1, used_piece_id = 30, piece_switch_interval = 86400000000, frozen_input_bytes = 13323961132, frozen_output_bytes = 13323961132, active_input_bytes = 117537394, active_output_bytes = 117537394, deleted_input_bytes = 0, deleted_output_bytes = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com', comment = ''", predicates="status = 'DOING' and round_id = 1 and checkpoint_scn = 1755223995192401003", affected_rows=1, sql=update __all_log_archive_progress set tenant_id = 1008, dest_no = 0, round_id = 1, incarnation = 1, dest_id = 1001, status = 'DOING', start_scn = 1752696006021581000, checkpoint_scn = 1755224040686907000, max_scn = 1755224115293438002, compatible = 1, base_piece_id = 1, used_piece_id = 30, piece_switch_interval = 86400000000, frozen_input_bytes = 13323961132, frozen_output_bytes = 13323961132, active_input_bytes = 117537394, active_output_bytes = 117537394, deleted_input_bytes = 0, deleted_output_bytes = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com', comment = '' where tenant_id = 1008 AND dest_no = 0 and status = 'DOING' and round_id = 1 and checkpoint_scn = 1755223995192401003) [2025-08-15 10:15:55.141543] WDIAG [STORAGE.TRANS] mvcc_sanity_check_ (ob_mvcc_row.cpp:937) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=24][errcode=-5024] find double insert node(ret=-5024, node=this=0x7f152f633f88 trans_version={val:4611686018427387903, v:0} scn={val:4611686018427387903, v:0} tx_id={txid:141877049} prev=(nil) next=(nil) modify_count=4294967295 acc_checksum=0 version=1755222899324351 type=0 flag=128 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:2082} write_epoch=2, prev=this=0x7f152f633d30 trans_version={val:1755224104878578000, v:0} scn={val:1755224104878578000, v:0} tx_id={txid:141876348} prev=0x7f152f633bb0 next=(nil) modify_count=49 acc_checksum=212852118 version=1755222899324351 type=0 flag=4 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:1062} write_epoch=4, snapshot_version={val:1755224155140728000, v:0}, *this={this=0x7f152f622458 latch_=locked flag=11 first_dml=DELETE last_dml=INSERT update_since_compact=2 list_head=0x7f152f633d30 latest_compact_node=0x7f152f6337d0 max_trans_version={val:1755224104878578000, v:0} max_trans_id=141876348 max_elr_trans_version={val:1755224104878578000, v:0} max_elr_trans_id=141876348 latest_compact_ts=1755224054673616 last_compact_cnt=47 total_trans_node_cnt=50 max_modify_scn={val:1755224104878578000, v:0} min_modify_scn={val:1755222899325101000, v:0}}) [2025-08-15 10:15:55.141578] WDIAG [STORAGE.TRANS] mvcc_write (ob_mvcc_row.cpp:999) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=27][errcode=-5024] mvcc write failed(ret=-5024, node=this=0x7f152f633f88 trans_version={val:4611686018427387903, v:0} scn={val:4611686018427387903, v:0} tx_id={txid:141877049} prev=(nil) next=(nil) modify_count=4294967295 acc_checksum=0 version=1755222899324351 type=0 flag=128 snapshot_barrier=0 snapshot_barrier_flag=0 mtd={dml_flag:3, buf_len:277} seq_no={branch:0, seq:2082} write_epoch=2, ctx={this:0x7f1a0fdc2dd0, ls_id:{id:1}, ls:0x7f1a567ca150, branch:0, timeout:1755224185041433, tablet_id:{id:350}, table_iter:0x7f1a0fdc1308, table_version:1746780010640136, mvcc_acc_ctx:{type:3, abs_lock_timeout_ts:1755224185041433, tx_lock_timeout_us:-1, snapshot:{version:{val:1755224155140728000, v:0}, tx_id:{txid:141877049}, scn:{branch:0, seq:2080}}, tx_table_guards:{tx_table_guard:{tx_table:0x7f1a567d5b10, epoch:1, mini_cache_:[0:{}, 1:{}, 2:{}, 3:{}]}, src_tx_table_guard:{tx_table:null, epoch:-1, mini_cache_:[0:{}, 1:{}, 2:{}, 3:{}]}, src_ls_handle:{ls_map_:null, ls_:null, mod_:-1}}, tx_id:{txid:141877049}, tx_desc:{this:0x7f1e79bf2520, tx_id:{txid:141877049}, state:2, addr:"10.203.1.34:2882", tenant_id:1007, session_id:3221609328, assoc_session_id:3221609328, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:{branch:0, seq:1}, op_sn:4, alloc_ts:1755224155140728, active_ts:1755224155140728, commit_ts:-1, finish_ts:-1, timeout_us:29999995, lock_timeout_us:-1, expire_ts:1755224185140723, coord_id:{id:-1}, parts:[{id:{id:1}, addr:"10.203.1.34:2882", epoch:509150327000707, first_scn:{branch:0, seq:3}, last_scn:{branch:0, seq:3}, last_touch_ts:1755224155138653}], exec_info_reap_ts:1755224155138652, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:1727264618, cluster_version:17180067073, seq_base:1755224155138649, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, conflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, modified_tables:[], last_rc_snapshot_version:{val:1755224155140728000, v:0}, ref:2}, tx_ctx:0x7f1aa3f1a750, mem_ctx:0x7f1aa3f1bcd0, tx_scn:{branch:0, seq:2082}, write_flag:{is_table_api:0, is_table_lock:0, is_mds:0, is_dml_batch_opt:0, is_insert_up:1, is_write_only_index:0, is_check_row_locked:0, is_lob_aux:0, is_skip_flush_redo:0, is_update_uk:0, is_update_pk_dop:0}, handle_start_time:-1, is_standby_read:false, lock_wait_start_ts:0, is_inited:true}, tablet_stat:{ls_id:0, tablet_id:0, query_cnt:0, merge_cnt:0, scan_logical_row_cnt:0, scan_physical_row_cnt:0, scan_micro_block_cnt:0, pushdown_micro_block_cnt:0, exist_row_total_table_cnt:0, exist_row_read_table_cnt:0, insert_row_cnt:0, update_row_cnt:0, delete_row_cnt:0}, is_read_store_ctx:false, update_full_column:false}) [2025-08-15 10:15:55.141660] WDIAG [STORAGE.TRANS] mvcc_write (ob_mvcc_engine.cpp:333) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=65][errcode=-5024] mvcc write failed(ret=-5024, arg={tx_id:{txid:141877049}, data:0x7f1a0fdbfcc0, old_row:null, modify_count:4294967295, acc_checksum:0, memstore_version:1755222899324351, seq_no:{branch:0, seq:2082}, write_epoch:2, scn:{val:4611686018427387903, v:0}, column_cnt:23}) [2025-08-15 10:15:55.141677] WDIAG [STORAGE.TRANS] on_key_duplication_retry (ob_memtable_context.cpp:314) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=13][errcode=0] primary key duplication conflict(key={"BIGINT":1008},{"BIGINT":1001},{"BIGINT":1},{"BIGINT":30}, this={ObIMvccCtx={alloc_type=0 ctx_descriptor=0 min_table_version=1746780010640088 max_table_version=1746780010640136 trans_version={val:4611686018427387903, v:0} commit_version={val:0, v:0} lock_wait_start_ts=0 replay_compact_version={val:0, v:0}} end_code=0 tx_status=0 is_readonly=false ref=1 trans_id={txid:141877049} ls_id=1 row_callback[alloc:1, free:0, unsubmit:1] redo[fill:0,sync_succ:0, sync_fail:0] main_list_len=3 pending_log_size=289 callback_list:{cnt=1 stat:[tx_end=0, rollback_to=0, fast_commit=0, remove_memtable=0, ext_info_log=0] detail:[(log_epoch,length,logged,synced,appended,removed,unlog_removed,branch_removed)|0:(1,3,0,0,3,0,0,0)|]}}) [2025-08-15 10:15:55.141700] WDIAG [STORAGE] insert_row_without_rowkey_check (ob_tablet.cpp:4643) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=18][errcode=-5024] fail to set memtable(ret=-5024) [2025-08-15 10:15:55.141711] WDIAG [STORAGE] insert_row_to_tablet (ob_ls_tablet_service.cpp:5667) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=8][errcode=-5024] failed to write table row(ret=-5024, table id=350, col_descs=[cnt:23, column_id=18 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=19 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=20 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=21 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=16 {type:"TIMESTAMP", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=17 {type:"TIMESTAMP", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=22 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=23 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=24 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=25 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=26 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=27 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=28 {type:"BIGINT UNSIGNED", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=29 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=30 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=31 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=32 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=33 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=34 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=35 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=36 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=37 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=38 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0], datum_row={row_flag:{flag:"INSERT", flag_type:0}, trans_id:{txid:0}, scan_index:0, mvcc_row_flag:{first:0, uncommitted:0, shadow:0, compact:0, ghost:0, last:0, reserved:0, flag:0}, snapshot_version:0, fast_filter_skipped:false, have_uncommited_row:false, group_idx:0, count:23, datum_buffer:{capacity:23, datums:0x7f1da5f13ea0, local_datums:0x7f1da5f13af0}, datums:[col_id=0:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138a4, hex: F003000000000000, int: 1008}, col_id=1:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138ac, hex: E903000000000000, int: 1001}, col_id=2:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138b4, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=3:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138bc, hex: 1E00000000000000, int: 30}, col_id=4:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138c4, hex: 78B244F85D3C0600, int: 1755224155140728}, col_id=5:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138cc, hex: 78B244F85D3C0600, int: 1755224155140728}, col_id=6:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138d4, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=7:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138dc, hex: 0000000000000000, int: 0}, col_id=8:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138e4, hex: 0000000000000000, int: 0}, col_id=9:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138ec, hex: C8881CAC8FBA5B18, int: 1755201606021581000}, col_id=10:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138f4, hex: 786E5E26F7CE5B18, int: 1755224040686907000}, col_id=11:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f138fc, hex: 32C0438508CF5B18, int: 1755224115293438002}, col_id=12:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f13904, hex: C8886B3D24095C18, int: 1755288006021581000}, col_id=13:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f1390c, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=14:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f13914, hex: 0100000000000000, int: 1, num_digit0: 0}, col_id=15:{len: 4, flag: 0, null: 0, ptr: 0x7f1da5f1391c, hex: 6E6F6E65, cstr: none}, col_id=16:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f13920, hex: 727A010700000000, int: 117537394}, col_id=17:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f13928, hex: 727A010700000000, int: 117537394}, col_id=18:{len: 6, flag: 0, null: 0, ptr: 0x7f1da5f13930, hex: 414354495645, cstr: ACTIVE}, col_id=19:{len: 9, flag: 0, null: 0, ptr: 0x7f1da5f13936, hex: 415641494C41424C45, cstr: AVAILABLE}, col_id=20:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f1393f, hex: 0000000000000000, int: 0}, col_id=21:{len: 8, flag: 0, null: 0, ptr: 0x7f1da5f13947, hex: 0000000000000000, int: 0}, col_id=22:{len: 135, flag: 0, null: 0, ptr: 0x7f1da5f1394f, hex: 636F733A2F2F6261636B75702D6F6365616E626173652D313235343233363236352F6261636B75702D6F62636C75737465722F6F625F766563746F722F313732373236343631382F74656E616E745F696E6361726E6174696F6E5F312F313030382F636C6F673F686F73743D636F732E61702D6265696A696E672E6D7971636C, cstr: cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcl}]}) [2025-08-15 10:15:55.141796] WDIAG [STORAGE] insert_row (ob_ls_tablet_service.cpp:3241) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=56][errcode=-5024] failed to write row(ret=-5024) [2025-08-15 10:15:55.141827] INFO [STORAGE.TRANS] rollback (ob_memtable_context.cpp:851) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=7] memtable handle rollback to successfuly(from_seq_no={branch:0, seq:3165}, to_seq_no={branch:0, seq:2081}, remove_cnt=1, elapsed=0, this={ObIMvccCtx={alloc_type=0 ctx_descriptor=0 min_table_version=1746780010640088 max_table_version=1746780010640136 trans_version={val:4611686018427387903, v:0} commit_version={val:0, v:0} lock_wait_start_ts=0 replay_compact_version={val:0, v:0}} end_code=0 tx_status=0 is_readonly=false ref=0 trans_id={txid:141877049} ls_id=1 row_callback[alloc:1, free:0, unsubmit:1] redo[fill:0,sync_succ:0, sync_fail:0] main_list_len=2 pending_log_size=289 callback_list:{cnt=1 stat:[tx_end=0, rollback_to=1, fast_commit=0, remove_memtable=0, ext_info_log=0] detail:[(log_epoch,length,logged,synced,appended,removed,unlog_removed,branch_removed)|0:(1,2,0,0,3,1,1,0)|]}}) [2025-08-15 10:15:55.142045] INFO [SHARE] do_insert_or_update_row_ (ob_inner_table_operator.cpp:699) [476034][T1007_ArchiveSv][T1007][YB420ACB0122-0006338212A554BF-0-0] [lt=15] insert/update one row(row={key:{tenant_id:1008, dest_id:1001, round_id:1, piece_id:30}, incarnation:1, dest_no:0, file_count:0, start_scn:{val:1755201606021581000, v:0}, checkpoint_scn:{val:1755224040686907000, v:0}, max_scn:{val:1755224115293438002, v:0}, end_scn:{val:1755288006021581000, v:0}, compatible:{version:1}, input_bytes:117537394, output_bytes:117537394, status:{status:"ACTIVE"}, file_status:0, cp_file_id:0, cp_file_offset:0, path:"cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com"}, affected_rows=2, sql=INSERT INTO __all_log_archive_piece_files (tenant_id, dest_id, round_id, piece_id, incarnation, dest_no, file_count, start_scn, checkpoint_scn, max_scn, end_scn, compatible, input_bytes, output_bytes, status, file_status, cp_file_id, cp_file_offset, path) VALUES (1008, 1001, 1, 30, 1, 0, 0, 1755201606021581000, 1755224040686907000, 1755224115293438002, 1755288006021581000, 1, 117537394, 117537394, 'ACTIVE', 'AVAILABLE', 0, 0, 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com') ON DUPLICATE KEY UPDATE incarnation = 1, dest_no = 0, file_count = 0, start_scn = 1755201606021581000, checkpoint_scn = 1755224040686907000, max_scn = 1755224115293438002, end_scn = 1755288006021581000, compatible = 1, input_bytes = 117537394, output_bytes = 117537394, status = 'ACTIVE', file_status = 'AVAILABLE', cp_file_id = 0, cp_file_offset = 0, path = 'cos://backup-oceanbase-1254236265/backup-obcluster/ob_vector/1727264618/tenant_incarnation_1/1008/clog?host=cos.ap-beijing.myqcloud.com') [2025-08-15 10:15:55.149417] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF066-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.168681] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62AE-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:55.171143] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62AE-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:55.186148] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D1D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.200706] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9D2-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.330809] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED5-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1007, thread_idx=0) [2025-08-15 10:15:55.332068] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1007, only_for_display=false) [2025-08-15 10:15:55.333303] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:55.347121] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.348279] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.349454] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.349478] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1007, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:55.349496] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.349508] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1007, ls_id:{id:1}, replicas:[{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1007, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:55.349612] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=80] build ls locality stat map(tenant_id=1007, ls_id={id:1}) [2025-08-15 10:15:55.349620] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.349633] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.349643] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.349650] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780009860497, create_time_us:1746780004427548, tenant_id:1007, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780005179445, create_time_us:1746780004424801, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780004442013, create_time_us:1746780004426211, tenant_id:1007, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:55.349728] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.349747] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=10] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.349756] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.349762] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.349771] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3819, tenant_id=1007, ls_id={id:1}) [2025-08-15 10:15:55.349781] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:55.350852] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18784, tenant_id=1007, acc_dr_task=0) [2025-08-15 10:15:55.352090] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1008, only_for_display=false) [2025-08-15 10:15:55.353297] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:55.367058] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.368216] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.369361] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.369387] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:55.369403] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.369415] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1}, replicas:[{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:55.369516] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=79] build ls locality stat map(tenant_id=1008, ls_id={id:1}) [2025-08-15 10:15:55.369525] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.369538] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.369551] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=10] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.369559] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780010617179, create_time_us:1746780010617179, tenant_id:1008, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010601653, create_time_us:1746780010601653, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010622320, create_time_us:1746780010622320, tenant_id:1008, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:55.369637] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.369651] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.369661] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.369666] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.369676] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3777, tenant_id=1008, ls_id={id:1}) [2025-08-15 10:15:55.369686] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:55.370776] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:55.373306] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1D-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.373615] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF75-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.375908] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1D-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:55.375926] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1D-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.376193] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF75-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:55.376208] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF75-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.391630] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.392792] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.393948] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.393972] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:55.393990] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.394001] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1001}, replicas:[{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:55.394103] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=80] build ls locality stat map(tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:15:55.394111] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.394124] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.394135] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.394147] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780016262199, create_time_us:1746780010802957, tenant_id:1008, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011098301, create_time_us:1746780010799498, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011101456, create_time_us:1746780010799175, tenant_id:1008, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:55.394225] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.394239] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.394248] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.394258] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.394265] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3783, tenant_id=1008, ls_id={id:1001}) [2025-08-15 10:15:55.394275] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:55.395334] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:55.416197] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.417347] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.418493] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.418517] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:55.418532] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.418541] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1002}, replicas:[{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:55.418647] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=83] build ls locality stat map(tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:15:55.418654] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.418666] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.418674] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.418683] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746780011865579, create_time_us:1746780010874431, tenant_id:1008, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780016393030, create_time_us:1746780010860516, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780010885508, create_time_us:1746780010862338, tenant_id:1008, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:55.418759] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.418772] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.418779] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.418786] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.418792] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3740, tenant_id=1008, ls_id={id:1002}) [2025-08-15 10:15:55.418800] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:55.419844] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:55.440672] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.441831] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.442004] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C687C-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.442990] INFO [RS] get (ob_server_table_operator.cpp:820) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:55.443010] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1008, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:55.443025] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.443034] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1008, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1008, ls_id:{id:1003}, replicas:[{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:55.443135] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=79] build ls locality stat map(tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:15:55.443143] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.443154] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.443162] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:55.443171] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746809744925556, create_time_us:1746780010917497, tenant_id:1008, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1011, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1011, in_pool:true, unit:unit_id:1011, resource_pool_id:1011, unit_group_id:1004, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746780011099354, create_time_us:1746780010915262, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1010, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1010, in_pool:true, unit:unit_id:1010, resource_pool_id:1010, unit_group_id:1004, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809745415104, create_time_us:1746780010917076, tenant_id:1008, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1012, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1012, in_pool:true, unit:unit_id:1012, resource_pool_id:1012, unit_group_id:1004, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:55.443247] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.443258] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.443265] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.443273] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:55.443279] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3745, tenant_id=1008, ls_id={id:1003}) [2025-08-15 10:15:55.443287] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:55.444424] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [476035][T1007_DRService][T1007][YB420ACB0122-00063382159F7ED6-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=93556, tenant_id=1008, acc_dr_task=0) [2025-08-15 10:15:55.461146] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:55.461157] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:55.461162] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=3] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:55.467924] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A090A-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.93164062GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}) [2025-08-15 10:15:55.467967] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A090A-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:55.485659] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785E-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.485976] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F0-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.485978] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C25-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.486122] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC057-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.488532] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CF-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.488868] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.488983] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F0-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.489056] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C25-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.489065] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC057-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.490842] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.490942] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785E-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.490968] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F0-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.491014] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC057-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.491077] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F0-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.491093] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C25-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.491207] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C25-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.491208] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC057-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.491471] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CF-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.493377] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CF-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.493477] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413CF-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.501772] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E732-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.501772] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0C3-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.533159] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:55.533172] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:55.533177] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:55.533765] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:55.533777] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:55.533788] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:55.539162] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:55.539176] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:55.539183] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:55.545274] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:55.545286] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:55.545292] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:55.550578] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:55.550592] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:55.550597] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:55.557195] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.560243] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764E-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.562124] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764E-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.562289] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764E-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.566024] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8858-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.568575] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8858-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:55.568592] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8858-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.571166] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A37-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.573674] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A37-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:55.573692] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A37-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.622098] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:55.623334] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224155622021, delay:5000000, job_id:0, tenant_id:1006}, job_key=0x7f1e7d8101e0, job_infos=[]) [2025-08-15 10:15:55.623358] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224160623356, delay:5000000, job_id:0, tenant_id:1006}) [2025-08-15 10:15:55.690816] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C687F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.700430] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [438021][T1006_TimerWK0_TntSharedTimer][T1006][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:55.702181] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD91-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.707408] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:55.707421] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:55.707429] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:55.751251] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDA0-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.751279] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF06A-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.753462] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC643-0-0] [lt=13] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.843679] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B1-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.846226] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B1-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:55.846244] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B1-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.848950] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D20-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.849578] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38CF-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:55.852160] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38CF-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:55.862463] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887D-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.865016] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887D-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:55.865033] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887D-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.896114] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB44-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.898627] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB44-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:55.898646] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB44-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:55.907008] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837F-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.910019] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837F-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.911960] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.912107] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D837F-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:55.927639] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B37A-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:55.946678] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:55.947899] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224155946637, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7f1e7d810120, job_infos=[]) [2025-08-15 10:15:55.947924] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224160947922, delay:5000000, job_id:0, tenant_id:1003}) [2025-08-15 10:15:55.971620] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F1-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:55.974097] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F1-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:55.975050] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5266-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:55.977514] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5266-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:55.992304] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58EB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:55.993075] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9CE-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.016642] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:56.017880] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224156007879, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7f1e7d810160, job_infos=[]) [2025-08-15 10:15:56.017901] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224161017899, delay:5000000, job_id:0, tenant_id:1004}) [2025-08-15 10:15:56.017916] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1755224161017914, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:56.050122] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB869C-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:56.052637] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB869C-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:56.053534] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5305-0-0] [lt=8] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:56.056089] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5305-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:56.104745] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58E5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.137057] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0C6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.137823] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FAB-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:56.140315] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FAB-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:56.149446] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF067-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.176449] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62AF-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:56.178948] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62AF-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:56.186224] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D1E-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.189563] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:56.190794] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224156189490, delay:5000000, job_id:0, tenant_id:1017}, job_key=0x7f1e7d8104a0, job_infos=[]) [2025-08-15 10:15:56.190817] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224161190816, delay:5000000, job_id:0, tenant_id:1017}) [2025-08-15 10:15:56.192035] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:56.193231] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224156191966, delay:5000000, job_id:0, tenant_id:1018}, job_key=0x7f1e7d8104e0, job_infos=[]) [2025-08-15 10:15:56.193250] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=14] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224161193249, delay:5000000, job_id:0, tenant_id:1018}) [2025-08-15 10:15:56.200776] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9D3-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.279256] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA1D-0-0] [lt=10] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.91992188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}) [2025-08-15 10:15:56.279297] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA1D-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=45, ddl_arg=NULL) [2025-08-15 10:15:56.378654] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1E-0-0] [lt=39] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.378947] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF76-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.381167] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1E-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:56.381183] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1E-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.381472] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF76-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:56.381488] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF76-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.442078] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C687D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.461131] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224154848160001, replayable_scn:1755224154848160001, readable_scn:1755224154848160001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224156457492, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224154848160001, replayable_scn:1755224154848160001, readable_scn:1755224154848160001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224156457492, ora_rowscn_=1755224155908515000, content_changed=true) [2025-08-15 10:15:56.461308] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:56.461318] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:56.461323] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:56.491977] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785F-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.492055] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F1-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.492084] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C26-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.492275] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC058-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.494485] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D0-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224156350271001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224155849572005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224156350271001, v:0}}], last_dump_ts_=1755224151463794, force_dump=false) [2025-08-15 10:15:56.494517] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D0-0-0] [lt=24] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.495146] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.495162] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F1-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.495254] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C26-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.495429] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC058-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224156350271001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224156350271001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224156350271002, v:0}}], last_dump_ts_=1755224151464025, force_dump=false) [2025-08-15 10:15:56.495452] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC058-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.497144] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785F-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.497153] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F1-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.497252] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB785F-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.497269] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F1-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.497430] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C26-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.497460] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC058-0-0] [lt=3] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.497530] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C26-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.497554] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC058-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.497599] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D0-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224156350271001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224155849572005, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224156350271001, v:0}}], last_dump_ts_=1755224151466741, force_dump=false) [2025-08-15 10:15:56.497622] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D0-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.499540] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D0-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.499725] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D0-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.501849] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0C4-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.501848] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E733-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.509048] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF21D-0-0] [lt=10] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.528643] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBA99-0-0] [lt=14] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.82617188GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83007812GB}) [2025-08-15 10:15:56.528682] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBA99-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=42, ddl_arg=NULL) [2025-08-15 10:15:56.533051] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224154469422000, replayable_scn:1755224154469422000, readable_scn:1755224154469422000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224156530147, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224154469422000, replayable_scn:1755224154469422000, readable_scn:1755224154469422000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224156530147, ora_rowscn_=1755224155559283000, content_changed=true) [2025-08-15 10:15:56.533239] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:56.533251] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:56.533257] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:56.533715] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224155348941001, replayable_scn:1755224155348941001, readable_scn:1755224155348941001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224156530147, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224155348941001, replayable_scn:1755224155348941001, readable_scn:1755224155348941001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224156530147, ora_rowscn_=1755224156496436000, content_changed=true) [2025-08-15 10:15:56.533889] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:56.533901] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:56.533915] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:56.539042] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224155348941004, replayable_scn:1755224155348941004, readable_scn:1755224155348941004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224156536468, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224155348941004, replayable_scn:1755224155348941004, readable_scn:1755224155348941004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224156536468, ora_rowscn_=1755224156494330000, content_changed=true) [2025-08-15 10:15:56.539219] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:56.539231] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:56.539236] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:56.545191] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224155250109000, replayable_scn:1755224155250109000, readable_scn:1755224155250109000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224156541696, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224155250109000, replayable_scn:1755224155250109000, readable_scn:1755224155250109000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224156541696, ora_rowscn_=1755224156494330000, content_changed=true) [2025-08-15 10:15:56.545361] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:56.545373] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:56.545378] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:56.550525] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224155348941002, replayable_scn:1755224155348941002, readable_scn:1755224155348941002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224156546965, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224155348941002, replayable_scn:1755224155348941002, readable_scn:1755224155348941002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224156546965, ora_rowscn_=1755224156494330000, content_changed=true) [2025-08-15 10:15:56.550697] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:56.550708] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:56.550714] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:56.563253] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764F-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.565659] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487710][T1003_DRService][T1003][YB420ACB0122-0006337FE6577D85-0-0] [lt=15] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=30000000, tenant_id=1003, thread_idx=1) [2025-08-15 10:15:56.566307] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764F-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.568172] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.568321] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D764F-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.569273] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [1487710][T1003_DRService][T1003][YB420ACB0122-0006337FE6577D86-0-0] [lt=15] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2374, tenant_id=1003, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:56.571299] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8859-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.571581] INFO [RS] manage_dr_tasks_ (ob_disaster_recovery_service.cpp:218) [1487710][T1003_DRService][T1003][YB420ACB0122-0006337FE6577D86-0-0] [lt=14] try manage dr tasks over(ret=0, ret="OB_SUCCESS", cost=2288, tenant_id=1004, need_clean_task=true, service_epoch_to_check=1) [2025-08-15 10:15:56.573831] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8859-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:56.573852] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D8859-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.576330] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A38-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.578855] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A38-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:56.578875] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A38-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.690894] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6880-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.702262] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD92-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.707305] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224155348941001, replayable_scn:1755224155348941001, readable_scn:1755224155348941001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224156703766, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224155348941001, replayable_scn:1755224155348941001, readable_scn:1755224155348941001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224156703766, ora_rowscn_=1755224156494330000, content_changed=true) [2025-08-15 10:15:56.707486] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:56.707497] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:56.707504] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:56.707515] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=6] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_time_us=188, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:56.751283] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDA1-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.751351] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF06B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.780953] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [709536][T1014_TimerWK3_TntSharedTimer][T1014][Y0-0000000000000000-0-0] [lt=11] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=1) [2025-08-15 10:15:56.807524] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [475908][T1008_TimerWK0_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=5] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:56.831003] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:56.832236] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224156830930, delay:5000000, job_id:0, tenant_id:1015}, job_key=0x7f1e7d810420, job_infos=[]) [2025-08-15 10:15:56.832263] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224161832260, delay:5000000, job_id:0, tenant_id:1015}) [2025-08-15 10:15:56.848954] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B2-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.849021] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D21-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.851480] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B2-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:56.851498] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B2-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.857555] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D0-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:56.860077] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D0-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:56.867913] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887E-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.870887] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887E-0-0] [lt=100] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:56.870906] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887E-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.901399] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB45-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.903902] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB45-0-0] [lt=65] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:56.903921] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB45-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:56.907859] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:115) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EAB6-0-0] [lt=11] [RESTORE] restore scheduler quit [2025-08-15 10:15:56.909141] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:198) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EAB6-0-0] [lt=12] get targets(ret=0, jobs=[], exec_tenant_id=1015, sql=select * from __all_recover_table_job) [2025-08-15 10:15:56.910394] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:108) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EAB7-0-0] [lt=15] get targets(ret=0, jobs=[], exec_tenant_id=1015, sql=select * from __all_import_table_job) [2025-08-15 10:15:56.910418] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:119) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EAB8-0-0] [lt=14][errcode=0] long time to hold one guard(schema_mgr=0x7f1e87cacf70, tenant_id=1, version=1750151511103928, cur_timestamp=1755224156910413, ref_timestamp=1755224096905639, lbt()="0x9510a36 0x1ba9c61f 0x90cf8b6 0x9192820 0x8da61c6 0xeb4902d 0xe19066d 0x2333c564 0x2334384a 0x23342f45 0x23340b2e 0x7f1f08c081cf 0x7f1f08839dd3") [2025-08-15 10:15:56.910506] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:84) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EAB9-0-0] [lt=15] [RESTORE] restore scheduler start [2025-08-15 10:15:56.910515] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:91) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EABA-0-0] [lt=7] [RESTORE] try process restore job [2025-08-15 10:15:56.911656] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:412) [727232][T1015_REST_SER][T1015][YB420ACB0122-000633A34ED7EABA-0-0] [lt=3] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2025-08-15 10:15:56.913099] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8380-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.916141] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8380-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.918055] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8380-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.918162] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8380-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:56.929069] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B37B-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:56.979455] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F2-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:56.981947] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F2-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:56.982766] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5267-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:56.985389] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5267-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:56.992378] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58EC-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:56.993148] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9CF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.058022] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB869D-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:57.061379] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5306-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:57.061781] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB869D-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:57.063870] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5306-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:57.104819] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58E6-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.137130] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0C7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.142972] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727134][T1016_TimerWK2_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=13] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:15:57.145695] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FAC-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:57.148187] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FAC-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:57.149521] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF068-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.184327] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B0-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:57.186299] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D1F-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.186817] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B0-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:57.200850] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9D4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.206691] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [727957][T1018_TimerWK1_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=14] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:57.207609] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:123) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=9] mview_mds_op(tx_count=0) [2025-08-15 10:15:57.207621] INFO [RS] update_mview_mds_op (ob_mview_mds_op_task.cpp:144) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=8] update_mview_mds(ret=0, ret="OB_SUCCESS", tx_set=, del_tx_id=[], cost=838) [2025-08-15 10:15:57.261007] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [2619485][T1012_TimerWK0_TntSharedTimer][T1012][Y0-0000000000000000-0-0] [lt=7] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=2) [2025-08-15 10:15:57.321325] INFO [RS] update_mview_refresh_info_cache (ob_mview_maintenance_service.cpp:301) [1487606][T1004_TimerWK3_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=75] update mview refresh info(ret=0, mview_ids=[], mview_refresh_scns=[], update_cache_cnt=0, del_mview_id=[], cost=3) [2025-08-15 10:15:57.335868] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:57.337114] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224157335790, delay:5000000, job_id:0, tenant_id:1016}, job_key=0x7f1e7d810460, job_infos=[]) [2025-08-15 10:15:57.337137] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224162337136, delay:5000000, job_id:0, tenant_id:1016}) [2025-08-15 10:15:57.367793] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1008, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227757366632}) [2025-08-15 10:15:57.367810] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1008, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:57.367819] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1008, category:1}) [2025-08-15 10:15:57.373710] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1016, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227757372544}) [2025-08-15 10:15:57.373726] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1016, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:57.373739] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1016, category:1}) [2025-08-15 10:15:57.374909] WDIAG [RS] register_or_update_storage_key_limit (ob_shared_storage_net_throt_service.cpp:737) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] failed to get storage limit(storage_key={storage_id_:1, tenant_id:1006, category:1}, limit={max_iops_:0, max_bw_:0, expire_time_:1755227757373752}) [2025-08-15 10:15:57.374924] WDIAG [RS] update_one_storage_quota_plan (ob_shared_storage_net_throt_service.cpp:418) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4201] SSNT:get storage iops and bandwidth limit failed(ret=-4201, storage_key={storage_id_:1, tenant_id:1006, category:1}, max_iops=200, max_bandwidth=25000000) [2025-08-15 10:15:57.374936] WDIAG [RS] update_quota_plan (ob_shared_storage_net_throt_service.cpp:403) [17349][TimerWK2_IngressService][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] SSNT:update quota plan failed(ret=0, tmp_ret=-4201, it->first={storage_id_:1, tenant_id:1006, category:1}) [2025-08-15 10:15:57.383866] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1F-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.384146] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF77-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.386438] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1F-0-0] [lt=71] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:57.386456] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A1F-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.386685] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF77-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:57.386703] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF77-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.422190] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:57.423421] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224157422114, delay:5000000, job_id:0, tenant_id:1013}, job_key=0x7f1e7d8103a0, job_infos=[]) [2025-08-15 10:15:57.423444] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224162423443, delay:5000000, job_id:0, tenant_id:1013}) [2025-08-15 10:15:57.431913] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E61-0-0] [lt=11] get number(ret=0, service_type_=0, range=1, base_id=1755224157431865000, start_id=1755224157431865000, end_id=1755224157431865001) [2025-08-15 10:15:57.432541] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E61-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224157431865000, v:0}, tenant_id=1017) [2025-08-15 10:15:57.432562] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [728181][T1017_FrzInfoDe][T1017][YB420ACB0122-000633A3555A7E61-0-0] [lt=12] zone_merge_mgr is already loaded(tenant_id=1017, global_merge_info={tenant_id:1017, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401596547, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194734514083, need_update:false}}, zone_merge_infos=[{tenant_id:1017, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194734514895, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401618002, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1017, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194734516645, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401622619, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1017, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194734518395, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401624780, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:15:57.442152] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C687E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.461278] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:57.461290] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:57.461296] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:57.468395] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A090B-0-0] [lt=7] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.87695312GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}) [2025-08-15 10:15:57.468433] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437495A090B-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=41, ddl_arg=NULL) [2025-08-15 10:15:57.498300] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F2-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.498300] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7860-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.498495] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C27-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.498488] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC059-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224157351811001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224157351811002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224157351811002, v:0}}], last_dump_ts_=1755224152467165, force_dump=false) [2025-08-15 10:15:57.498512] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC059-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.500707] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F2-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.500730] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D1-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.501222] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7860-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224157351811003, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224157351811003, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224157351811003, v:0}}], last_dump_ts_=1755224152470328, force_dump=false) [2025-08-15 10:15:57.501246] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7860-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.501469] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C27-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.501701] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC059-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.501924] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0C5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.501927] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E734-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.502657] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F2-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.502757] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F2-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.503283] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7860-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.503366] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7860-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.503480] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C27-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.503568] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C27-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.504158] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC059-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.504258] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC059-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.504311] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D1-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.506272] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D1-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.506363] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D1-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.533290] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:57.533302] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:57.533307] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:57.533949] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:57.533959] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:57.533964] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:57.535242] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1012, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:57.539800] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:57.539819] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:57.539827] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:57.545369] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:57.545383] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:57.545388] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:57.546657] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1008, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:57.550729] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:57.550740] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:57.550745] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:57.566568] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E6A-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224157566379000, start_id=1755224157566379000, end_id=1755224157566379001) [2025-08-15 10:15:57.567208] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727352][T1015_FrzInfoDe][T1015][YB420ACB0122-000633A34ECA7E6A-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224157566379000, v:0}, tenant_id=1015) [2025-08-15 10:15:57.567731] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66C-0-0] [lt=63] get number(ret=0, service_type_=0, range=1, base_id=1755224157567432000, start_id=1755224157567432000, end_id=1755224157567432001) [2025-08-15 10:15:57.568408] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438234][T1005_FrzInfoDe][T1005][YB420ACB0122-0006338206EFA66C-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224157567432000, v:0}, tenant_id=1005) [2025-08-15 10:15:57.569271] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7650-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.572109] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7650-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.573944] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7650-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.574074] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7650-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.576578] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885A-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.579174] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885A-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:57.579190] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885A-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.581555] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A39-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.584072] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A39-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:57.584087] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A39-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.690967] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6881-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.702342] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD93-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.707476] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:57.707492] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:57.707498] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:57.751354] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDA2-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.751422] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF06C-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.758376] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC644-0-0] [lt=14] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.795958] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36CA-0-0] [lt=12] get number(ret=0, service_type_=0, range=1, base_id=1755224157795654000, start_id=1755224157795654000, end_id=1755224157795654001) [2025-08-15 10:15:57.796581] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619732][T1011_FrzInfoDe][T1011][YB420ACB0122-0006338CC11C36CA-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224157795654000, v:0}, tenant_id=1011) [2025-08-15 10:15:57.800646] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:57.801912] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224157800574, delay:5000000, job_id:0, tenant_id:1014}, job_key=0x7f1e7d8103e0, job_infos=[]) [2025-08-15 10:15:57.801936] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224162801934, delay:5000000, job_id:0, tenant_id:1014}) [2025-08-15 10:15:57.803146] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224157801766, delay:5000000, job_id:0, tenant_id:1011}, job_key=0x7f1e7d810320, job_infos=[]) [2025-08-15 10:15:57.803167] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224162803166, delay:5000000, job_id:0, tenant_id:1011}) [2025-08-15 10:15:57.849090] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D22-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.854218] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B3-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.854366] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=9] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:57.854400] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=23] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:57.854416] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [475909][T1008_TimerWK1_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=12] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:57.855657] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=13] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:57.855685] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=20] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:57.855699] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [1487603][T1004_TimerWK0_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=11] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:57.856800] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B3-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:57.856819] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B3-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.856872] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [475911][T1008_TimerWK3_TntSharedTimer][T1008][Y0-0000000000000000-0-0] [lt=7] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:57.856888] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [1487604][T1004_TimerWK1_TntSharedTimer][T1004][Y0-0000000000000000-0-0] [lt=15] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:57.865375] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D1-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:57.867884] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D1-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:57.873636] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887F-0-0] [lt=39] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.876231] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887F-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:57.876250] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD887F-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.906794] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB46-0-0] [lt=36] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.909345] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB46-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:57.909362] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB46-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:57.913952] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB2-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224157913430000, start_id=1755224157913430000, end_id=1755224157913430001) [2025-08-15 10:15:57.914595] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [709758][T1013_FrzInfoDe][T1013][YB420ACB0122-000633A33E5A7FB2-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224157913430000, v:0}, tenant_id=1013) [2025-08-15 10:15:57.919178] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8381-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.922391] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:454) [727997][T1018_STSWorker][T1018][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1755224157536942, tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224156350271001, replayable_scn:1755224156350271001, readable_scn:1755224156350271001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-08-15 10:15:57.922771] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8381-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.924999] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8381-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.925135] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8381-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:57.925386] INFO [RS] prepare (ob_mview_maintenance_task.cpp:176) [727134][T1016_TimerWK2_TntSharedTimer][T1016][Y0-0000000000000000-0-0] [lt=14] mview maintenance task prepare success(tenant_id_=1016, round_=63409, prepare_cost_us_=0) [2025-08-15 10:15:57.930504] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B37C-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:57.964039] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:57.965259] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224157963989, delay:5000000, job_id:0, tenant_id:1012}, job_key=0x7f1e7d810360, job_infos=[]) [2025-08-15 10:15:57.965283] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224162965281, delay:5000000, job_id:0, tenant_id:1012}) [2025-08-15 10:15:57.987389] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F3-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:57.990750] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5268-0-0] [lt=14] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:57.991111] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F3-0-0] [lt=14] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:57.992455] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58ED-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.993228] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9D0-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:57.993284] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5268-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:58.049751] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:58.051019] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224158049675, delay:5000000, job_id:0, tenant_id:1007}, job_key=0x7f1e7d810220, job_infos=[]) [2025-08-15 10:15:58.051041] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224163051040, delay:5000000, job_id:0, tenant_id:1007}) [2025-08-15 10:15:58.068738] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB869E-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:58.071539] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5307-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:58.072413] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB869E-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:58.074279] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5307-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:58.092968] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B2-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224158092370000, start_id=1755224158092370000, end_id=1755224158092370001) [2025-08-15 10:15:58.093833] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [2619736][T1012_FrzInfoDe][T1012][YB420ACB0122-0006338CC1EC36B2-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224158092370000, v:0}, tenant_id=1012) [2025-08-15 10:15:58.104895] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58E7-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.112798] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463BA-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224158112373000, start_id=1755224158112373000, end_id=1755224158112373001) [2025-08-15 10:15:58.113642] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [17916][T1_FrzInfoDet][T1][YB420ACB0122-0006341128A463BA-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224158112373000, v:0}, tenant_id=1) [2025-08-15 10:15:58.137205] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0C8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.149598] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF069-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.153586] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FAD-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:58.156111] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FAD-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:58.186371] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D20-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.192038] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B1-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:58.194566] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B1-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:58.200922] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9D5-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.279710] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA1E-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92382812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.92773438GB}) [2025-08-15 10:15:58.279755] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17377][LeaseQueueTh1][T0][YB420ACB0426-000634286BF9FA1E-0-0] [lt=27] [DDL] execute ddl like stmt(ret=0, cost=49, ddl_arg=NULL) [2025-08-15 10:15:58.320798] INFO [RS] run2 (ob_archive_scheduler_service.cpp:79) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A630-0-0] [lt=9] start do ObArchiveSchedulerService round(round=124005) [2025-08-15 10:15:58.324385] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:950) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A630-0-0] [lt=14][errcode=-4018] no row exist(ret=-4018) [2025-08-15 10:15:58.324400] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A630-0-0] [lt=11][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1012 AND dest_no = 0 AND name = 'state', exec_tenant_id=1011) [2025-08-15 10:15:58.324463] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A630-0-0] [lt=10][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1012, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:58.324481] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A630-0-0] [lt=14][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1012, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2025-08-15 10:15:58.324496] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:404) [2619611][T1011_ArchiveSv][T1011][YB420ACB0122-0006338CC154A630-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1012, *item:{dest_no:0, name:"state", *value:{value:}}}) [2025-08-15 10:15:58.357795] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8271-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1005, thread_idx=0) [2025-08-15 10:15:58.359099] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1005, only_for_display=false) [2025-08-15 10:15:58.360327] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:58.367316] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [728182][T1017_MergeSche][T1017][YB420ACB0122-000633A355E2D5E4-0-0] [lt=60] zone_merge_mgr is already loaded(tenant_id=1017, global_merge_info={tenant_id:1017, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401596547, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194734514083, need_update:false}}, zone_merge_infos=[{tenant_id:1017, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194734514895, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401618002, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1017, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194734516645, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401622619, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1017, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194734518395, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194401624780, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194401569429000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:15:58.374044] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.375186] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.376345] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.376368] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1005, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:58.376384] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.376395] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1005, ls_id:{id:1}, replicas:[{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1005, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:58.376497] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=79] build ls locality stat map(tenant_id=1005, ls_id={id:1}) [2025-08-15 10:15:58.376510] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.376523] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.376535] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.376546] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770631175955, create_time_us:1746770625668071, tenant_id:1005, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770626348462, create_time_us:1746770625662213, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770625679132, create_time_us:1746770625662260, tenant_id:1005, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:58.376622] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.376636] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.376646] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.376651] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.376658] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3764, tenant_id=1005, ls_id={id:1}) [2025-08-15 10:15:58.376668] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:58.377796] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18697, tenant_id=1005, acc_dr_task=0) [2025-08-15 10:15:58.379011] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1006, only_for_display=false) [2025-08-15 10:15:58.380199] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:58.389389] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF78-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.389511] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A20-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.391952] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF78-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:58.391967] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF78-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.392044] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A20-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:58.392060] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A20-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.393804] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.394955] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.396098] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.396126] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:58.396140] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.396149] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1}, replicas:[{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:58.396250] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=79] build ls locality stat map(tenant_id=1006, ls_id={id:1}) [2025-08-15 10:15:58.396257] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.396268] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.396276] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.396284] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770632068744, create_time_us:1746770632068744, tenant_id:1006, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083541, create_time_us:1746770632083541, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770632083151, create_time_us:1746770632083151, tenant_id:1006, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:58.396359] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.396370] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.396377] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.396384] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.396391] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3743, tenant_id=1006, ls_id={id:1}) [2025-08-15 10:15:58.396399] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:58.397466] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:58.418063] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=10] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.419210] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.420351] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=15] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.420377] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:58.420393] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=9] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.420406] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1001}, replicas:[{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:58.420516] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=86] build ls locality stat map(tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:15:58.420523] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.420535] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.420543] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=6] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.420551] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770637963853, create_time_us:1746770632437593, tenant_id:1006, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633276285, create_time_us:1746770632473923, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632495890, tenant_id:1006, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:58.420626] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=55] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.420638] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.420645] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.420653] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.420660] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3766, tenant_id=1006, ls_id={id:1001}) [2025-08-15 10:15:58.420668] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:58.421770] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:58.442228] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C687F-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.442350] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.443497] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=24] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.444631] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.444654] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:58.444676] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=14] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.444688] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1002}, replicas:[{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:58.444789] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=79] build ls locality stat map(tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:15:58.444797] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.444810] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.444817] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.444827] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746770641146296, create_time_us:1746770632632531, tenant_id:1006, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770641572301, create_time_us:1746770632674064, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746770633277627, create_time_us:1746770632691604, tenant_id:1006, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:58.444904] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.444918] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.444928] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.444933] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.444946] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=10] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3737, tenant_id=1006, ls_id={id:1002}) [2025-08-15 10:15:58.444953] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=4] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:58.446066] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:58.461311] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:58.461324] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:58.461330] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:58.466802] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=11] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.467977] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.469125] INFO [RS] get (ob_server_table_operator.cpp:820) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.469148] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1006, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:58.469164] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.469176] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1006, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1006, ls_id:{id:1003}, replicas:[{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:58.469279] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=80] build ls locality stat map(tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:15:58.469289] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.469302] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=9] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.469313] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.469324] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806605261342, create_time_us:1746770632747345, tenant_id:1006, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1008, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1008, in_pool:true, unit:unit_id:1008, resource_pool_id:1008, unit_group_id:1003, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724783169, create_time_us:1746770632774152, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1009, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1009, in_pool:true, unit:unit_id:1009, resource_pool_id:1009, unit_group_id:1003, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809725275629, create_time_us:1746770632793733, tenant_id:1006, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:18, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1007, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1007, in_pool:true, unit:unit_id:1007, resource_pool_id:1007, unit_group_id:1003, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:58.469401] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.469415] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.469425] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.469433] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.469439] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=5] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3808, tenant_id=1006, ls_id={id:1003}) [2025-08-15 10:15:58.469449] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:58.470534] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [438127][T1005_DRService][T1005][YB420ACB0122-0006338209DF8272-0-0] [lt=4] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=92721, tenant_id=1006, acc_dr_task=0) [2025-08-15 10:15:58.501995] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0C6-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.502006] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E735-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.503718] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F3-0-0] [lt=7] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224158354083001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224158354083001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224158354083001, v:0}}], last_dump_ts_=1755224153473266, force_dump=false) [2025-08-15 10:15:58.503749] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F3-0-0] [lt=23] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.504288] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7861-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.504541] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C28-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.505186] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.506466] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F3-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224158354083001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224158354083001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224158354083001, v:0}}], last_dump_ts_=1755224153476539, force_dump=false) [2025-08-15 10:15:58.506489] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F3-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.507260] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D2-0-0] [lt=5] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.507327] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7861-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.507545] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C28-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.508201] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05A-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.508476] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F3-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.508592] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F3-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.509256] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7861-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.509330] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:262) [17733][T1_HBService][T1][YB420ACB0122-00063411183AF21E-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.509358] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7861-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.509500] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C28-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.509597] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C28-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.510146] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D2-0-0] [lt=14] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.510240] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05A-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.510332] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05A-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.512009] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D2-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.512132] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D2-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.529117] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBA9B-0-0] [lt=8] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83007812GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.83398438GB}) [2025-08-15 10:15:58.529159] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17378][LeaseQueueTh2][T0][YB420ACB0427-0006341E5FADBA9B-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=46, ddl_arg=NULL) [2025-08-15 10:15:58.533292] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:58.533304] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:58.533309] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:58.533971] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:58.533990] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:58.533997] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:58.539891] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:58.539903] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:58.539909] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:58.545412] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:58.545423] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:58.545428] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:58.551323] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:58.551341] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=14] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:58.551347] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:58.575071] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7651-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.578167] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7651-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.580249] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7651-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.580346] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7651-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.581905] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885B-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.584426] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885B-0-0] [lt=62] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:58.584442] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885B-0-0] [lt=12] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.586800] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3A-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.589380] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3A-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:58.589399] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3A-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.607782] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:58.609041] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224158607711, delay:5000000, job_id:0, tenant_id:1008}, job_key=0x7f1e7d810260, job_infos=[]) [2025-08-15 10:15:58.609065] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224163609064, delay:5000000, job_id:0, tenant_id:1008}) [2025-08-15 10:15:58.632829] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF0-0-0] [lt=10] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1, thread_idx=0) [2025-08-15 10:15:58.634090] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-08-15 10:15:58.635318] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:58.649096] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.650273] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.651440] INFO [RS] get (ob_server_table_operator.cpp:820) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:58.651463] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:58.651480] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.651492] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:58.651600] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=85] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-08-15 10:15:58.651610] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.651626] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=13] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.651637] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:58.651645] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=6] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745470458404419, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745470458625011, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1, unit_group_id:1, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746806409000209, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1, unit_group_id:1, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:58.651722] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.651736] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.651746] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.651750] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:58.651769] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=14] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3841, tenant_id=1, ls_id={id:1}) [2025-08-15 10:15:58.651781] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:58.652851] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [17715][T1_DRService][T1][YB420ACB0122-00063411BE945DF1-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18761, tenant_id=1, acc_dr_task=0) [2025-08-15 10:15:58.691040] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6882-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.702414] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD94-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.707535] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:58.707549] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:58.707554] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:58.749342] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5A-0-0] [lt=13] get number(ret=0, service_type_=0, range=1, base_id=1755224158748477000, start_id=1755224158748477000, end_id=1755224158748477001) [2025-08-15 10:15:58.749961] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [727355][T1016_FrzInfoDe][T1016][YB420ACB0122-000633A34F9A7E5A-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224158748477000, v:0}, tenant_id=1016) [2025-08-15 10:15:58.751429] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDA3-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.751495] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF06D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.763530] INFO [RS] publish_scn (ob_replica_safe_check_task.cpp:325) [727956][T1018_TimerWK0_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=13] publish_scn finish(ret=0, ret="OB_SUCCESS", need_publish=false, major_mv_merge_scn={val:18446744073709551615, v:3}, lastest_merge_scn={val:18446744073709551615, v:3}, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}, ls_attr_array=[]) [2025-08-15 10:15:58.763565] INFO [RS] switch_status (ob_replica_safe_check_task.cpp:193) [727956][T1018_TimerWK0_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=23] replica safe check task switch_status(ret=0, ret="OB_SUCCESS", new_status=0, delay=30000000, this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:58.763580] INFO [RS] runTimerTask (ob_replica_safe_check_task.cpp:173) [727956][T1018_TimerWK0_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=12] timer task finish(ret=0, ret="OB_SUCCESS", this={status:0, in_sched:true, is_stop:false, is_inited:true, merge_scn:{val:18446744073709551615, v:3}, max_transfer_task_id:{id:-1}, ls_cache:{arr_.count():0, arr:[]}}) [2025-08-15 10:15:58.765010] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:455) [727958][T1018_TimerWK2_TntSharedTimer][T1018][Y0-0000000000000000-0-0] [lt=24] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2025-08-15 10:15:58.849164] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D23-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.859511] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B4-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.862025] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B4-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:58.862043] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B4-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.873227] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D2-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:58.875733] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D2-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:58.879060] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8880-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.881650] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8880-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:58.881669] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8880-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.912159] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB47-0-0] [lt=35] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.914716] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB47-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:58.914736] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB47-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:58.926195] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8382-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.929370] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8382-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224158853597001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224158853597001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224158853597001, v:0}}], last_dump_ts_=1755224153897870, force_dump=false) [2025-08-15 10:15:58.929396] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8382-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.931514] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8382-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.931625] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8382-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:58.931933] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B37D-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:58.992527] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58EE-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.993261] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9D1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:58.996382] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F4-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1006, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:58.998527] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5269-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1009, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:58.998848] INFO [RS] do_work (ob_common_ls_service.cpp:128) [2619601][T1011_COMMONLSS][T1011][YB420ACB0122-000634393BF3E8F4-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:59.001487] INFO [RS] do_work (ob_common_ls_service.cpp:128) [728051][T1017_COMMONLSS][T1017][YB420ACB0122-000634205CAD5269-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:59.079277] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB869F-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1003, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:59.081946] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5308-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1008, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:59.082932] INFO [RS] do_work (ob_common_ls_service.cpp:128) [438116][T1005_COMMONLSS][T1005][YB420ACB0122-00063424DCCB869F-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:59.084708] INFO [RS] do_work (ob_common_ls_service.cpp:128) [727227][T1015_COMMONLSS][T1015][YB420ACB0122-000634205BCD5308-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:59.102199] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [1487819][T1004_MergeSche][T1004][YB420ACB0122-0006337FE442DBFE-0-0] [lt=71] zone_merge_mgr is already loaded(tenant_id=1004, global_merge_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402914553, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194735253569, need_update:false}}, zone_merge_infos=[{tenant_id:1004, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194735255910, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403065502, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1004, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194735259562, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403073022, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1004, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194735261458, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194403077657, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402689919000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:15:59.104972] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487570][T1004_StandbySc][T1004][YB420ACB0122-000634225C5C58E8-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.137284] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727226][T1015_StandbySc][T1015][YB420ACB0122-0006341EFFFDF0C9-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.149670] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [728050][T1017_StandbySc][T1017][YB420ACB0122-0006341F00ADF06A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.161434] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FAE-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1007, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:59.163960] INFO [RS] do_work (ob_common_ls_service.cpp:128) [709625][T1013_COMMONLSS][T1013][YB420ACB0122-0006342044CD5FAE-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:59.164619] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64D-0-0] [lt=18] get number(ret=0, service_type_=0, range=1, base_id=1755224159163831000, start_id=1755224159163831000, end_id=1755224159163831001) [2025-08-15 10:15:59.165521] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64D-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224159163831000, v:0}, tenant_id=1006) [2025-08-15 10:15:59.165544] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:134) [438237][T1006_FrzInfoDe][T1006][YB420ACB0122-0006338207BFA64D-0-0] [lt=15] zone_merge_mgr is already loaded(tenant_id=1006, global_merge_info={tenant_id:1006, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402201372, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194784919589, need_update:false}}, zone_merge_infos=[{tenant_id:1006, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194784921725, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402224483, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1006, zone:"zone2", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194784925063, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402226936, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}, {tenant_id:1006, zone:"zone3", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194784926922, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1755194402229083, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1755194402051618000, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2025-08-15 10:15:59.186446] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [437988][T1006_StandbySc][T1006][YB420ACB0122-000634221BDC8D21-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.199878] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B2-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1004, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:59.200991] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619600][T1011_StandbySc][T1011][YB420ACB0122-000634376D54A9D6-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.202385] INFO [RS] do_work (ob_common_ls_service.cpp:128) [476024][T1007_COMMONLSS][T1007][YB420ACB0122-0006342518CB62B2-0-0] [lt=12] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:59.228888] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A6-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224159228018000, start_id=1755224159228018000, end_id=1755224159228018001) [2025-08-15 10:15:59.229700] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [476158][T1008_FrzInfoDe][T1008][YB420ACB0122-00063382132FA2A6-0-0] [lt=15] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224159228018000, v:0}, tenant_id=1008) [2025-08-15 10:15:59.328056] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E63-0-0] [lt=9] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1003, thread_idx=0) [2025-08-15 10:15:59.329329] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1003, only_for_display=false) [2025-08-15 10:15:59.330556] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:59.344454] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.345606] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.346780] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.346803] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1003, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:59.346820] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.346833] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1003, ls_id:{id:1}, replicas:[{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:59.346936] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=81] build ls locality stat map(tenant_id=1003, ls_id={id:1}) [2025-08-15 10:15:59.346949] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.346964] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=11] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.346976] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.346988] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023130649, create_time_us:1745916017668426, tenant_id:1003, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017681590, create_time_us:1745916017665781, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916017683049, create_time_us:1745916017667275, tenant_id:1003, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:59.347066] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.347080] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.347089] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.347095] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=5] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.347104] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3807, tenant_id=1003, ls_id={id:1}) [2025-08-15 10:15:59.347115] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:59.348178] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=3] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=18849, tenant_id=1003, acc_dr_task=0) [2025-08-15 10:15:59.349398] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1004, only_for_display=false) [2025-08-15 10:15:59.350579] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=13] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:59.364451] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.365628] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.366823] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.366846] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:59.366863] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.366875] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:59.366988] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=90] build ls locality stat map(tenant_id=1004, ls_id={id:1}) [2025-08-15 10:15:59.366996] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.367009] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.367020] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.367030] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916023847505, create_time_us:1745916023847505, tenant_id:1004, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023850295, create_time_us:1745916023850295, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916023851708, create_time_us:1745916023851708, tenant_id:1004, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:59.367110] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.367124] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.367134] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.367143] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.367149] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3847, tenant_id=1004, ls_id={id:1}) [2025-08-15 10:15:59.367160] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:59.368231] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:59.388964] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.390139] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.391324] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.391348] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:59.391366] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.391377] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:59.391481] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=82] build ls locality stat map(tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:15:59.391490] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.391503] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.391509] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.391521] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916029464612, create_time_us:1745916024031604, tenant_id:1004, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024324215, create_time_us:1745916024062037, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024326141, create_time_us:1745916024081791, tenant_id:1004, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:59.391607] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=63] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.391623] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.391632] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.391643] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.391653] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3834, tenant_id=1004, ls_id={id:1001}) [2025-08-15 10:15:59.391663] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:59.392742] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:59.394702] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF79-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1006, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1006, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1006, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632364933001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1006, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632371263001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1006, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1003, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746770632375484001, v:0}}}], tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.394726] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A21-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023954532000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023961902001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1745916023969271000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.397252] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A21-0-0] [lt=63] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-08-15 10:15:59.397263] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF79-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1006) [2025-08-15 10:15:59.397271] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [1487573][T1004_PLSSer][T1004][YB420ACB0122-00063424A4EB9A21-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1745916032827008, tenant_name:"opsone", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.397278] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [437991][T1006_PLSSer][T1006][YB420ACB0122-000634236F6BDF79-0-0] [lt=11] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1006, schema_version:1746770641342592, tenant_name:"test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.413552] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.414728] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=22] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.415895] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.415919] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:59.415933] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.415951] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=13] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1002}, replicas:[{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:59.416061] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=86] build ls locality stat map(tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:15:59.416069] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.416082] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.416089] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=5] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.416100] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1745916032692127, create_time_us:1745916024176344, tenant_id:1004, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916033170840, create_time_us:1745916024177863, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024200784, create_time_us:1745916024182877, tenant_id:1004, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:59.416175] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.416190] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.416200] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.416205] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.416210] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=4] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3819, tenant_id=1004, ls_id={id:1002}) [2025-08-15 10:15:59.416220] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:59.417285] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:59.438700] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=12] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.439841] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.440990] INFO [RS] get (ob_server_table_operator.cpp:820) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.441014] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:59.441031] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.441038] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=5] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1003}, replicas:[{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:59.441147] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=82] build ls locality stat map(tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:15:59.441155] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.441168] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.441179] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.441187] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1746806607231050, create_time_us:1745916024220684, tenant_id:1004, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1006, unit_group_id:1002, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1745916024281125, create_time_us:1745916024262156, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1004, unit_group_id:1002, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1746809724772094, create_time_us:1745916024220797, tenant_id:1004, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1005, unit_group_id:1002, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:59.441264] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=58] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.441278] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.441287] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.441296] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.441306] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3746, tenant_id=1004, ls_id={id:1003}) [2025-08-15 10:15:59.441312] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=5] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:59.442306] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [475875][T1008_StandbySc][T1008][YB420ACB0122-000634224D1C6880-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.442394] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [1487709][T1003_DRService][T1003][YB420ACB0122-0006342706338E64-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=94201, tenant_id=1004, acc_dr_task=0) [2025-08-15 10:15:59.461219] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224157852343001, replayable_scn:1755224157852343001, readable_scn:1755224157852343001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224159458315, tenant_id=1014, tenant_info_={tenant_id:1014, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224157852343001, replayable_scn:1755224157852343001, readable_scn:1755224157852343001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224159458315, ora_rowscn_=1755224158928312000, content_changed=true) [2025-08-15 10:15:59.461398] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:59.461408] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:59.461415] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:59.462685] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [709500][T1014_TenantInf][T1014][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1014, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:59.468865] INFO [RS] receive_hb (ob_server_manager.cpp:653) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A08C0-0-0] [lt=9] server resource changed(old_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88085938GB}, new_resource_info={cpu_capacity:48, cpu_assigned:21, cpu_assigned_max:21, mem_capacity:"137.746464GB", mem_assigned:"39GB", mem_in_use:0GB, log_disk_capacity:2681.375GB, log_disk_assigned:111.491733GB, log_disk_in_use:89.8125GB, data_disk_capacity:1340.71875GB, data_disk_assigned:0GB, data_disk_in_use:2.88476562GB}) [2025-08-15 10:15:59.468904] INFO [RS] process_ (ob_rs_rpc_processor.h:236) [17376][LeaseQueueTh0][T0][YB420ACB0122-00063437577A08C0-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=43, ddl_arg=NULL) [2025-08-15 10:15:59.502077] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [17703][T1_StandbySchem][T1][YB420ACB0122-000634340E85E736-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.502073] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727099][T1016_StandbySc][T1016][YB420ACB0122-0006341F000DF0C7-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.509576] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F4-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.510314] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7862-0-0] [lt=8] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224159354201002, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224159354201002, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224159354201002, v:0}}], last_dump_ts_=1755224154479513, force_dump=false) [2025-08-15 10:15:59.510338] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7862-0-0] [lt=18] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.510535] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C29-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224159261633000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224159261633000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224159261633000, v:0}}], last_dump_ts_=1755224154479754, force_dump=false) [2025-08-15 10:15:59.510558] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C29-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.511300] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05B-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.512168] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F4-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.513137] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D3-0-0] [lt=6] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.513532] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7862-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.513726] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C29-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224159354201001, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224159354201001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224159354201001, v:0}}], last_dump_ts_=1755224154482809, force_dump=false) [2025-08-15 10:15:59.513747] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C29-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.514226] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F4-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.514321] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727924][T1018_TeRec][T1018][YB420ACB0122-000634202F3D76F4-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.514403] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05B-0-0] [lt=15] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.515554] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7862-0-0] [lt=8] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.515642] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [1487571][T1004_TeRec][T1004][YB420ACB0122-0006342504BB7862-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.515856] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C29-0-0] [lt=7] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.515946] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [475876][T1008_TeRec][T1008][YB420ACB0122-000634248F5B9C29-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.516161] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D3-0-0] [lt=13] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.516320] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.516424] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [437989][T1006_TeRec][T1006][YB420ACB0122-000634241F3BC05B-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.518053] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D3-0-0] [lt=11] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.518167] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [2619453][T1012_TeRec][T1012][YB420ACB0122-000634390D8413D3-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.533183] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224157455960000, replayable_scn:1755224157455960000, readable_scn:1755224157455960000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224159529827, tenant_id=1016, tenant_info_={tenant_id:1016, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224157455960000, replayable_scn:1755224157455960000, readable_scn:1755224157455960000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224159529827, ora_rowscn_=1755224158577079000, content_changed=true) [2025-08-15 10:15:59.533346] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:59.533356] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:59.533363] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:59.533959] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224158353029001, replayable_scn:1755224158353029001, readable_scn:1755224158353029001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224159530882, tenant_id=1012, tenant_info_={tenant_id:1012, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224158353029001, replayable_scn:1755224158353029001, readable_scn:1755224158353029001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224159530882, ora_rowscn_=1755224159515131000, content_changed=true) [2025-08-15 10:15:59.534146] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:59.534159] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:59.534165] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [2619450][T1012_TenantInf][T1012][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:59.534631] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727097][T1016_TenantInf][T1016][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1016, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:59.539782] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224158353029004, replayable_scn:1755224158353029004, readable_scn:1755224158353029004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224159537162, tenant_id=1018, tenant_info_={tenant_id:1018, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224158353029004, replayable_scn:1755224158353029004, readable_scn:1755224158353029004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224159537162, ora_rowscn_=1755224159510949000, content_changed=true) [2025-08-15 10:15:59.539979] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:59.539992] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:59.539997] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:59.541261] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [727921][T1018_TenantInf][T1018][Y0-0000000000000000-0-0] [lt=4] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1018, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:59.545330] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224158353029002, replayable_scn:1755224158353029002, readable_scn:1755224158353029002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224159542428, tenant_id=1008, tenant_info_={tenant_id:1008, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224158353029002, replayable_scn:1755224158353029002, readable_scn:1755224158353029002, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224159542428, ora_rowscn_=1755224159512003000, content_changed=true) [2025-08-15 10:15:59.545502] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=16] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:59.545512] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:59.545520] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [475873][T1008_TenantInf][T1008][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:59.550729] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224158353029002, replayable_scn:1755224158353029002, readable_scn:1755224158353029002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224159547695, tenant_id=1006, tenant_info_={tenant_id:1006, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224158353029002, replayable_scn:1755224158353029002, readable_scn:1755224158353029002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224159547695, ora_rowscn_=1755224159513056000, content_changed=true) [2025-08-15 10:15:59.550948] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=15] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:59.550958] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:59.550966] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:59.552246] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [437986][T1006_TenantInf][T1006][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1006, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:59.581342] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7652-0-0] [lt=6] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224159388882000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224159388882001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224159388882001, v:0}}], last_dump_ts_=1755224154550932, force_dump=false) [2025-08-15 10:15:59.581365] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7652-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.584552] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7652-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224159388882000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224159388882001, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224159388882001, v:0}}], last_dump_ts_=1755224154553856, force_dump=false) [2025-08-15 10:15:59.584581] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7652-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.586709] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7652-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.586836] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [727100][T1016_TeRec][T1016][YB420ACB0122-00063420314D7652-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.587135] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885C-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1018, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501740084001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501748519000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151501752746001, v:0}}}], tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.589717] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885C-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1018) [2025-08-15 10:15:59.589736] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727926][T1018_PLSSer][T1018][YB420ACB0122-0006341FFB0D885C-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1018, schema_version:1750151511102616, tenant_name:"media", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.592054] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3B-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1012, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1012, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1012, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525924072000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1012, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525932493000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1012, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1006, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1749023525936716001, v:0}}}], tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.594568] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3B-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1012) [2025-08-15 10:15:59.594587] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [2619455][T1012_PLSSer][T1012][YB420ACB0122-00063438A6042A3B-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1012, schema_version:1749023533610720, tenant_name:"oms", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.628426] INFO [RS] scheduler (ob_dbms_job_master.cpp:318) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2025-08-15 10:15:59.629659] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:603) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224159628349, delay:5000000, job_id:0, tenant_id:1005}, job_key=0x7f1e7d8101a0, job_infos=[]) [2025-08-15 10:15:59.629682] INFO [RS] scheduler (ob_dbms_job_master.cpp:331) [17462][DBMS_JOB_MASTER][T0][YB420ACB0122-0006337F0262C1CD-0-0] [lt=17] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1755224164629681, delay:5000000, job_id:0, tenant_id:1005}) [2025-08-15 10:15:59.662697] INFO [RS] do_dr_service_work_ (ob_disaster_recovery_service.cpp:136) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BC-0-0] [lt=14] [DRTASK_NOTICE] disaster recovery service finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, tenant_id=1017, thread_idx=0) [2025-08-15 10:15:59.664010] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1017, only_for_display=false) [2025-08-15 10:15:59.665290] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:59.679516] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=16] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.680680] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=20] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.681868] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.681892] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1017, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:59.681909] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.681921] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1017, ls_id:{id:1}, replicas:[{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1017, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:59.682077] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=122] build ls locality stat map(tenant_id=1017, ls_id={id:1}) [2025-08-15 10:15:59.682085] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.682099] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.682109] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.682115] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=5] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151500903141, create_time_us:1750151495444923, tenant_id:1017, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151495459542, create_time_us:1750151495444808, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151496129922, create_time_us:1750151495443213, tenant_id:1017, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:59.682193] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.682207] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.682216] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.682221] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.682230] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3956, tenant_id=1017, ls_id={id:1}) [2025-08-15 10:15:59.682239] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:59.683388] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=19378, tenant_id=1017, acc_dr_task=0) [2025-08-15 10:15:59.684595] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:1994) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1018, only_for_display=false) [2025-08-15 10:15:59.685846] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:59.691115] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [476023][T1007_StandbySc][T1007][YB420ACB0122-000634224D0C6883-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.699825] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.701000] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.702156] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=18] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.702178] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1}, acc_dr_task=0) [2025-08-15 10:15:59.702195] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.702206] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1}, replicas:[{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:59.702310] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=81] build ls locality stat map(tenant_id=1018, ls_id={id:1}) [2025-08-15 10:15:59.702318] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=4] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.702331] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.702343] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=9] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.702353] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151501666824, create_time_us:1750151501666824, tenant_id:1018, ls_id:{id:1}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501670308, create_time_us:1750151501670308, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151501680094, create_time_us:1750151501680094, tenant_id:1018, ls_id:{id:1}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:59.702430] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.702445] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.702455] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.702460] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.702469] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3835, tenant_id=1018, ls_id={id:1}) [2025-08-15 10:15:59.702478] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:59.702491] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709502][T1014_StandbySc][T1014][YB420ACB0122-0006341EF2BDFD95-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.703553] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:59.707445] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:785) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224158353029002, replayable_scn:1755224158353029002, readable_scn:1755224158353029002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1755224159704280, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1755224158353029002, replayable_scn:1755224158353029002, readable_scn:1755224158353029002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1755224159704280, ora_rowscn_=1755224159512003000, content_changed=true) [2025-08-15 10:15:59.707633] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.203.1.34:2882") [2025-08-15 10:15:59.707645] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.203.4.38:2882") [2025-08-15 10:15:59.707651] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:407) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.203.4.39:2882") [2025-08-15 10:15:59.707663] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:416) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=7] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_time_us=194, return_code_array=[0, 0, 0], broadcast_times=1) [2025-08-15 10:15:59.708954] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1023) [1487568][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=0, all_service_names_=[]) [2025-08-15 10:15:59.723713] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E6-0-0] [lt=14] get number(ret=0, service_type_=0, range=1, base_id=1755224159723168000, start_id=1755224159723168000, end_id=1755224159723168001) [2025-08-15 10:15:59.724426] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487815][T1003_FrzInfoDe][T1003][YB420ACB0122-000634227293B5E6-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224159723168000, v:0}, tenant_id=1003) [2025-08-15 10:15:59.724834] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.726082] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.727253] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.727278] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1001}, acc_dr_task=0) [2025-08-15 10:15:59.727295] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=11] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.727307] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1001}, replicas:[{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:59.727422] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=90] build ls locality stat map(tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:15:59.727430] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.727443] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.727453] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.727464] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151507264375, create_time_us:1750151501816002, tenant_id:1018, ls_id:{id:1001}, server:"10.203.1.34:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502057055, create_time_us:1750151501870993, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502034617, create_time_us:1750151501872569, tenant_id:1018, ls_id:{id:1001}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:59.727542] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=57] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.727556] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.727566] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.727576] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.727581] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=3] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3940, tenant_id=1018, ls_id={id:1001}) [2025-08-15 10:15:59.727592] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=8] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:59.728699] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:59.749885] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=13] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.751078] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=19] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.751500] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [709624][T1013_StandbySc][T1013][YB420ACB0122-0006341EF28DFDA4-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.751569] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [727923][T1018_StandbySc][T1018][YB420ACB0122-0006341F009DF06E-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.752252] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=21] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.752275] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1002}, acc_dr_task=0) [2025-08-15 10:15:59.752291] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=10] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.752303] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1002}, replicas:[{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:59.752405] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:15:59.752413] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.752426] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.752437] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.752448] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510460192, create_time_us:1750151501936154, tenant_id:1018, ls_id:{id:1002}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151510967011, create_time_us:1750151501976073, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.38:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151503037660, create_time_us:1750151501984117, tenant_id:1018, ls_id:{id:1002}, server:"10.203.4.39:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:59.752525] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=56] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.752540] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.752549] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.752554] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.752562] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=6] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3829, tenant_id=1018, ls_id={id:1002}) [2025-08-15 10:15:59.752570] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=6] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:59.753628] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2042) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-08-15 10:15:59.763196] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:394) [17734][T1_HBService][T1][YB420ACB0122-00063411DB7AC645-0-0] [lt=15] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.774926] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=14] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.1.34:2882", server_info_in_table={server:"10.203.1.34:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470459440089, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.776122] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=26] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.38:2882", server_info_in_table={server:"10.203.4.38:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470460620817, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.777286] INFO [RS] get (ob_server_table_operator.cpp:820) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=17] get server from table(ret=0, ret="OB_SUCCESS", server="10.203.4.39:2882", server_info_in_table={server:"10.203.4.39:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.1_101000042025031818-b6d5706eb3d2c5f501c7fa646ddbf32f3dc87069(Mar 18 2025 18:13:36)", stop_time:0, start_service_time:1745470464804828, block_migrate_in_time:0, last_offline_time:0}) [2025-08-15 10:15:59.777310] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3672) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1018, ls_id={id:1003}, acc_dr_task=0) [2025-08-15 10:15:59.777331] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3930) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=12] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.777342] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4444) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1018, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1018, ls_id:{id:1003}, replicas:[{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1018, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1009, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-08-15 10:15:59.777445] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:258) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=80] build ls locality stat map(tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:15:59.777454] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.777467] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=10] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.777477] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:363) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=8] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-08-15 10:15:59.777489] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:406) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1750151510614052, create_time_us:1750151502109003, tenant_id:1018, ls_id:{id:1003}, server:"10.203.1.34:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1026, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1026, in_pool:true, unit:unit_id:1026, resource_pool_id:1026, unit_group_id:1009, zone:"zone1", server:"10.203.1.34:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.1.34:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151502172854, create_time_us:1750151502155966, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.38:2882", sql_port:2881, role:2, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1027, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1027, in_pool:true, unit:unit_id:1027, resource_pool_id:1027, unit_group_id:1009, zone:"zone2", server:"10.203.4.38:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.38:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1750151511081364, create_time_us:1750151502110430, tenant_id:1018, ls_id:{id:1003}, server:"10.203.4.39:2882", sql_port:2881, role:1, member_list:[{server:"10.203.1.34:2882", timestamp:1}, {server:"10.203.4.38:2882", timestamp:1}, {server:"10.203.4.39:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1025, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1025, in_pool:true, unit:unit_id:1025, resource_pool_id:1025, unit_group_id:1009, zone:"zone3", server:"10.203.4.39:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.203.4.39:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-08-15 10:15:59.777573] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4493) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=61] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.777586] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4641) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=6] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.777595] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5337) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.777601] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5525) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-08-15 10:15:59.777610] INFO [RS.LB] check_ls_disaster_recovery_tasks_ (ob_disaster_recovery_worker.cpp:2158) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=7] finish check dr tasks(ret=0, ret="OB_SUCCESS", cost=3873, tenant_id=1018, ls_id={id:1003}) [2025-08-15 10:15:59.777621] INFO [RS.LB] try_ls_disaster_recovery (ob_disaster_recovery_worker.cpp:2064) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=9] skip persist dr_task(only_for_display=false, task_count=0) [2025-08-15 10:15:59.778762] INFO [RS] try_tenant_disaster_recovery_ (ob_disaster_recovery_service.cpp:190) [728062][T1017_DRService][T1017][YB420ACB0122-000633A3587A68BD-0-0] [lt=5] try check dr tasks over(ret=0, ret="OB_SUCCESS", cost=95358, tenant_id=1018, acc_dr_task=0) [2025-08-15 10:15:59.849233] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [438115][T1005_StandbySc][T1005][YB420ACB0122-000634221BCC8D24-0-0] [lt=10] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.864450] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:384) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C3-0-0] [lt=15] get number(ret=0, service_type_=0, range=1, base_id=1755224159864073000, start_id=1755224159864073000, end_id=1755224159864073001) [2025-08-15 10:15:59.864736] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B5-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1014, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1014, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1014, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136000240001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1014, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136007626000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1014, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1007, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750148136011843001, v:0}}}], tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.865242] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:348) [1487818][T1004_FrzInfoDe][T1004][YB420ACB0122-000634227953B5C3-0-0] [lt=13] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1755224159864073000, v:0}, tenant_id=1004) [2025-08-15 10:15:59.867346] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B5-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1014) [2025-08-15 10:15:59.867364] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [709505][T1014_PLSSer][T1014][YB420ACB0122-0006341FB84D95B5-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1014, schema_version:1750148137991032, tenant_name:"grafana_test", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.881136] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1229) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D3-0-0] [lt=9] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-08-15 10:15:59.883660] INFO [RS] do_work (ob_common_ls_service.cpp:128) [1487698][T1003_COMMONLSS][T1003][YB420ACB0122-00063425756B38D3-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:59.884410] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8881-0-0] [lt=29] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1016, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1016, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1016, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411713138001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1016, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411719470000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1016, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1008, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1750151411723691001, v:0}}}], tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.887007] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8881-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1016) [2025-08-15 10:15:59.887025] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [727102][T1016_PLSSer][T1016][YB420ACB0122-0006341FF9FD8881-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1016, schema_version:1750151421180008, tenant_name:"temporalio", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.917511] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:130) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB48-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1008, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1008, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1008, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010728270001, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1008, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010734592001, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1008, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1004, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1746780010738769001, v:0}}}], tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.920065] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:462) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB48-0-0] [lt=64] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1008) [2025-08-15 10:15:59.920084] INFO [RS] do_work (ob_primary_ls_service.cpp:88) [475878][T1008_PLSSer][T1008][YB420ACB0122-000634244C8BBB48-0-0] [lt=15] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1008, schema_version:1746780018525208, tenant_name:"mcp", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-08-15 10:15:59.932617] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:788) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8383-0-0] [lt=9] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.203.1.34:2882", readable_scn:{val:1755224159855646000, v:0}}, {server:"10.203.4.38:2882", readable_scn:{val:1755224159855646000, v:0}}, {server:"10.203.4.39:2882", readable_scn:{val:1755224159855646000, v:0}}], last_dump_ts_=1755224154900908, force_dump=false) [2025-08-15 10:15:59.932645] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8383-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.933362] INFO [RS] do_work (ob_common_ls_service.cpp:128) [17704][T1_COMMONLSSe][T1][YB420ACB0122-0006343470D5B37E-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-08-15 10:15:59.935888] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8383-0-0] [lt=16] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.937958] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8383-0-0] [lt=9] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.938112] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:108) [709503][T1014_TeRec][T1014][YB420ACB0122-000634200A1D8383-0-0] [lt=12] is primary tenant, no need check readable_scn [2025-08-15 10:15:59.992599] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [1487697][T1003_StandbySc][T1003][YB420ACB0122-000634225C1C58EF-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-08-15 10:15:59.993335] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:68) [2619452][T1012_StandbySc][T1012][YB420ACB0122-000634376D64A9D2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS")